From a01e26a39baf0908cd30351c7f17e6a19d00276b Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@gmail.com>
Date: Fri, 13 Jul 2018 13:13:21 +0100
Subject: [PATCH 001/260] Correct spelling of
 AnalysisPlugin#requriesAnalysisSettings (#32025)

Because this is a static method on a public API, and one that we encourage
plugin authors to use, the method with the typo is deprecated in 6.x
rather than just renamed.
---
 .../migration/migrate_7_0/plugins.asciidoc    |  5 ++++
 .../analysis/common/CommonAnalysisPlugin.java | 26 +++++++++----------
 .../indices/analysis/AnalysisModule.java      |  4 +--
 .../elasticsearch/plugins/AnalysisPlugin.java | 20 ++++++++++----
 4 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/docs/reference/migration/migrate_7_0/plugins.asciidoc b/docs/reference/migration/migrate_7_0/plugins.asciidoc
index 829a93573c9..f8434993078 100644
--- a/docs/reference/migration/migrate_7_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_7_0/plugins.asciidoc
@@ -18,3 +18,8 @@ See {plugins}/repository-azure-repository-settings.html#repository-azure-reposit
 must now be specified in the client settings instead.
 
 See {plugins}/repository-gcs-client.html#repository-gcs-client[Google Cloud Storage Client Settings].
+
+==== Analysis Plugin changes
+
+* The misspelled helper method `requriesAnalysisSettings(AnalyzerProvider<T> provider)` has been
+renamed to `requiresAnalysisSettings`
\ No newline at end of file
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
index ca2f74b5efe..d95af920a30 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
@@ -135,7 +135,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
-import static org.elasticsearch.plugins.AnalysisPlugin.requriesAnalysisSettings;
+import static org.elasticsearch.plugins.AnalysisPlugin.requiresAnalysisSettings;
 
 public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin {
 
@@ -201,11 +201,11 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin {
         filters.put("cjk_width", CJKWidthFilterFactory::new);
         filters.put("classic", ClassicFilterFactory::new);
         filters.put("czech_stem", CzechStemTokenFilterFactory::new);
-        filters.put("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new));
+        filters.put("common_grams", requiresAnalysisSettings(CommonGramsTokenFilterFactory::new));
         filters.put("decimal_digit", DecimalDigitFilterFactory::new);
         filters.put("delimited_payload_filter", LegacyDelimitedPayloadTokenFilterFactory::new);
         filters.put("delimited_payload", DelimitedPayloadTokenFilterFactory::new);
-        filters.put("dictionary_decompounder", requriesAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new));
+        filters.put("dictionary_decompounder", requiresAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new));
         filters.put("dutch_stem", DutchStemTokenFilterFactory::new);
         filters.put("edge_ngram", EdgeNGramTokenFilterFactory::new);
         filters.put("edgeNGram", EdgeNGramTokenFilterFactory::new);
@@ -216,11 +216,11 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin {
         filters.put("german_normalization", GermanNormalizationFilterFactory::new);
         filters.put("german_stem", GermanStemTokenFilterFactory::new);
         filters.put("hindi_normalization", HindiNormalizationFilterFactory::new);
-        filters.put("hyphenation_decompounder", requriesAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new));
+        filters.put("hyphenation_decompounder", requiresAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new));
         filters.put("indic_normalization", IndicNormalizationFilterFactory::new);
-        filters.put("keep", requriesAnalysisSettings(KeepWordFilterFactory::new));
-        filters.put("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new));
-        filters.put("keyword_marker", requriesAnalysisSettings(KeywordMarkerTokenFilterFactory::new));
+        filters.put("keep", requiresAnalysisSettings(KeepWordFilterFactory::new));
+        filters.put("keep_types", requiresAnalysisSettings(KeepTypesFilterFactory::new));
+        filters.put("keyword_marker", requiresAnalysisSettings(KeywordMarkerTokenFilterFactory::new));
         filters.put("kstem", KStemTokenFilterFactory::new);
         filters.put("length", LengthTokenFilterFactory::new);
         filters.put("limit", LimitTokenCountFilterFactory::new);
@@ -229,8 +229,8 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin {
         filters.put("multiplexer", MultiplexerTokenFilterFactory::new);
         filters.put("ngram", NGramTokenFilterFactory::new);
         filters.put("nGram", NGramTokenFilterFactory::new);
-        filters.put("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
-        filters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new));
+        filters.put("pattern_capture", requiresAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
+        filters.put("pattern_replace", requiresAnalysisSettings(PatternReplaceTokenFilterFactory::new));
         filters.put("persian_normalization", PersianNormalizationFilterFactory::new);
         filters.put("porter_stem", PorterStemTokenFilterFactory::new);
         filters.put("remove_duplicates", RemoveDuplicatesTokenFilterFactory::new);
@@ -241,10 +241,10 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin {
         filters.put("serbian_normalization", SerbianNormalizationFilterFactory::new);
         filters.put("snowball", SnowballTokenFilterFactory::new);
         filters.put("sorani_normalization", SoraniNormalizationFilterFactory::new);
-        filters.put("stemmer_override", requriesAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
+        filters.put("stemmer_override", requiresAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
         filters.put("stemmer", StemmerTokenFilterFactory::new);
         filters.put("trim", TrimTokenFilterFactory::new);
-        filters.put("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new));
+        filters.put("truncate", requiresAnalysisSettings(TruncateTokenFilterFactory::new));
         filters.put("unique", UniqueTokenFilterFactory::new);
         filters.put("uppercase", UpperCaseTokenFilterFactory::new);
         filters.put("word_delimiter_graph", WordDelimiterGraphTokenFilterFactory::new);
@@ -256,8 +256,8 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin {
     public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
         Map<String, AnalysisProvider<CharFilterFactory>> filters = new TreeMap<>();
         filters.put("html_strip", HtmlStripCharFilterFactory::new);
-        filters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new));
-        filters.put("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new));
+        filters.put("pattern_replace", requiresAnalysisSettings(PatternReplaceCharFilterFactory::new));
+        filters.put("mapping", requiresAnalysisSettings(MappingCharFilterFactory::new));
         return filters;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
index 364732dc183..1ecdc797073 100644
--- a/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
+++ b/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -54,7 +54,7 @@ import java.util.Locale;
 import java.util.Map;
 
 import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.plugins.AnalysisPlugin.requriesAnalysisSettings;
+import static org.elasticsearch.plugins.AnalysisPlugin.requiresAnalysisSettings;
 
 /**
  * Sets up {@link AnalysisRegistry}.
@@ -118,7 +118,7 @@ public final class AnalysisModule {
         tokenFilters.register("stop", StopTokenFilterFactory::new);
         tokenFilters.register("standard", StandardTokenFilterFactory::new);
         tokenFilters.register("shingle", ShingleTokenFilterFactory::new);
-        tokenFilters.register("hunspell", requriesAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory
+        tokenFilters.register("hunspell", requiresAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory
             (indexSettings, name, settings, hunspellService)));
 
         tokenFilters.extractAndRegister(plugins, AnalysisPlugin::getTokenFilters);
diff --git a/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java b/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
index e740fddc6ec..27b5667d3bb 100644
--- a/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
@@ -57,12 +57,12 @@ import static java.util.Collections.emptyMap;
  * }</pre>
  *
  * Elasticsearch doesn't have any automatic mechanism to share these components between indexes. If any component is heavy enough to warrant
- * such sharing then it is the Pugin's responsibility to do it in their {@link AnalysisProvider} implementation. We recommend against doing
+ * such sharing then it is the Plugin's responsibility to do it in their {@link AnalysisProvider} implementation. We recommend against doing
  * this unless absolutely necessary because it can be difficult to get the caching right given things like behavior changes across versions.
  */
 public interface AnalysisPlugin {
     /**
-     * Override to add additional {@link CharFilter}s. See {@link #requriesAnalysisSettings(AnalysisProvider)}
+     * Override to add additional {@link CharFilter}s. See {@link #requiresAnalysisSettings(AnalysisProvider)}
      * how to on get the configuration from the index.
      */
     default Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
@@ -70,7 +70,7 @@ public interface AnalysisPlugin {
     }
 
     /**
-     * Override to add additional {@link TokenFilter}s. See {@link #requriesAnalysisSettings(AnalysisProvider)}
+     * Override to add additional {@link TokenFilter}s. See {@link #requiresAnalysisSettings(AnalysisProvider)}
      * how to on get the configuration from the index.
      */
     default Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
@@ -78,7 +78,7 @@ public interface AnalysisPlugin {
     }
 
     /**
-     * Override to add additional {@link Tokenizer}s. See {@link #requriesAnalysisSettings(AnalysisProvider)}
+     * Override to add additional {@link Tokenizer}s. See {@link #requiresAnalysisSettings(AnalysisProvider)}
      * how to on get the configuration from the index.
      */
     default Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
@@ -86,7 +86,7 @@ public interface AnalysisPlugin {
     }
 
     /**
-     * Override to add additional {@link Analyzer}s. See {@link #requriesAnalysisSettings(AnalysisProvider)}
+     * Override to add additional {@link Analyzer}s. See {@link #requiresAnalysisSettings(AnalysisProvider)}
      * how to on get the configuration from the index.
      */
     default Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
@@ -130,8 +130,18 @@ public interface AnalysisPlugin {
 
     /**
      * Mark an {@link AnalysisProvider} as requiring the index's settings.
+     *
+     * @deprecated use {@link #requiresAnalysisSettings(AnalysisProvider)}
      */
+    @Deprecated
     static <T> AnalysisProvider<T> requriesAnalysisSettings(AnalysisProvider<T> provider) {
+        return requiresAnalysisSettings(provider);
+    }
+
+    /**
+     * Mark an {@link AnalysisProvider} as requiring the index's settings.
+     */
+    static <T> AnalysisProvider<T> requiresAnalysisSettings(AnalysisProvider<T> provider) {
         return new AnalysisProvider<T>() {
             @Override
             public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {

From 92a9bb2b965fb401543aedf71ea7707da55e1a15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Fri, 13 Jul 2018 14:34:02 +0200
Subject: [PATCH 002/260] Re-instate link in StringFunctionUtils javadocs

The previous errors in compileJava were not cause by the brackets but my the
content of the @link section. Corrected this so its a working javadoc link again.
---
 .../function/scalar/string/StringFunctionUtils.java           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
index 75db52ed099..cef826d37ed 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java
@@ -8,7 +8,7 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.string;
 abstract class StringFunctionUtils {
 
     /**
-     * Trims the trailing whitespace characters from the given String. Uses @link java.lang.Character.isWhitespace(char)
+     * Trims the trailing whitespace characters from the given String. Uses {@link Character#isWhitespace(char)}
      * to determine if a character is whitespace or not.
      *
      * @param s       the original String
@@ -27,7 +27,7 @@ abstract class StringFunctionUtils {
     }
 
     /**
-     * Trims the leading whitespace characters from the given String. Uses @link java.lang.Character.isWhitespace(char)
+     * Trims the leading whitespace characters from the given String. Uses {@link Character#isWhitespace(char)}
      * to determine if a character is whitespace or not.
      *
      * @param s       the original String

From 9a928756e979e6f2f1469aa00b89c2c050788d68 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Fri, 13 Jul 2018 15:40:38 +0200
Subject: [PATCH 003/260] Docs: Change formatting of Cloud options

---
 docs/reference/setup/install.asciidoc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc
index 7675e5ad146..4433ffb8c38 100644
--- a/docs/reference/setup/install.asciidoc
+++ b/docs/reference/setup/install.asciidoc
@@ -1,11 +1,16 @@
 [[install-elasticsearch]]
 == Installing Elasticsearch
 
+[float]
+=== Hosted Elasticsearch
 Elasticsearch can be run on your own hardware or using our hosted
 Elasticsearch Service on https://www.elastic.co/cloud[Elastic Cloud], which is
 available on AWS and GCP. You can
 https://www.elastic.co/cloud/elasticsearch-service/signup[try out the hosted service] for free.
 
+[float]
+=== Installing Elasticsearch Yourself
+
 Elasticsearch is provided in the following package formats:
 
 [horizontal]

From bc1284eb28a68cc2b5d18a3181ac72ff12804348 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Fri, 13 Jul 2018 15:48:14 +0200
Subject: [PATCH 004/260] Docs: Restyled cloud link in getting started

---
 docs/reference/getting-started.asciidoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index a29a743fed8..ff00c310a43 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -104,10 +104,13 @@ With that out of the way, let's get started with the fun part...
 
 == Installation
 
+[TIP]
+==============
 You can skip installation completely by using our hosted
 Elasticsearch Service on https://www.elastic.co/cloud[Elastic Cloud], which is
 available on AWS and GCP. You can
 https://www.elastic.co/cloud/elasticsearch-service/signup[try out the hosted service] for free.
+==============
 
 Elasticsearch requires at least Java 8. Specifically as of this writing, it is recommended that you use the Oracle JDK version {jdk}. Java installation varies from platform to platform so we won't go into those details here. Oracle's recommended installation documentation can be found on http://docs.oracle.com/javase/8/docs/technotes/guides/install/install_overview.html[Oracle's website]. Suffice to say, before you install Elasticsearch, please check your Java version first by running (and then install/upgrade accordingly if needed):
 

From b7f07f03edb9c5be2665d65eafc8fe9ad95d0fa1 Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Fri, 13 Jul 2018 10:07:42 -0400
Subject: [PATCH 005/260] [Rollup] Use composite's missing_bucket (#31402)

We can leverage the composite agg's new `missing_bucket` feature on
terms groupings.  This means the aggregation criteria used in the indexer
will now return null buckets for missing keys.

Because all buckets are now returned (even if a key is null),
we can guarantee correct doc counts with
"combined" jobs (where a job rolls up multiple schemas).  This was
previously impossible since composite would ignore documents that
didn't have _all_ the keys, meaning non-overlapping schemas would
cause composite to return no buckets.

Note: date_histo does not use `missing_bucket`, since a timestamp is
always required.

The docs have been adjusted to recommend a single, combined job.  It
also makes reference to the previous issue to help users that are upgrading
(rather than just deleting the sections).
---
 .../en/rollup/understanding-groups.asciidoc   | 237 +-----------------
 .../core/rollup/job/DateHistoGroupConfig.java |   1 -
 .../core/rollup/job/HistoGroupConfig.java     |   1 +
 .../core/rollup/job/TermsGroupConfig.java     |   1 +
 .../rollup/RollupRestTestStateCleaner.java    |  24 +-
 .../rollup/RollupResponseTranslator.java      |   1 +
 .../action/TransportPutRollupJobAction.java   |   8 +
 .../xpack/rollup/job/IndexerUtils.java        |  15 +-
 .../xpack/rollup/job/RollupIndexer.java       |   1 +
 .../RollupResponseTranslationTests.java       |  48 ++++
 .../action/PutJobStateMachineTests.java       |  55 +++-
 .../xpack/rollup/job/IndexerUtilsTests.java   | 106 +++++++-
 .../xpack/test/rest/XPackRestIT.java          |   2 +-
 .../xpack/restart/FullClusterRestartIT.java   |  61 ++++-
 14 files changed, 298 insertions(+), 263 deletions(-)

diff --git a/x-pack/docs/en/rollup/understanding-groups.asciidoc b/x-pack/docs/en/rollup/understanding-groups.asciidoc
index f57f905ae04..803555b2d73 100644
--- a/x-pack/docs/en/rollup/understanding-groups.asciidoc
+++ b/x-pack/docs/en/rollup/understanding-groups.asciidoc
@@ -121,16 +121,15 @@ if a field is useful for aggregating later, and how you might wish to use it (te
 
 === Grouping Limitations with heterogeneous indices
 
-There is a known limitation to Rollup groups, due to some internal implementation details at this time.  The Rollup feature leverages
-the `composite` aggregation from Elasticsearch.  At the moment, the composite agg only returns buckets when all keys in the tuple are non-null.
-Put another way, if the you request keys `[A,B,C]` in the composite aggregation, the only documents that are aggregated are those that have
-_all_ of the keys `A, B` and `C`.
+There was previously a limitation in how Rollup could handle indices that had heterogeneous mappings (multiple, unrelated/non-overlapping
+mappings).  The recommendation at the time was to configure a separate job per data "type".  For example, you might configure a separate
+job for each Beats module that you had enabled (one for `process`, another for `filesystem`, etc).
 
-Because Rollup uses the composite agg during the indexing process, it inherits this behavior.  Practically speaking, if all of the documents
-in your index are homogeneous (they have the same mapping), you can ignore this limitation and stop reading now.
+This recommendation was driven by internal implementation details that caused document counts to be potentially incorrect if a single "merged"
+job was used.
 
-However, if you have a heterogeneous collection of documents that you wish to roll up, you may need to configure two or more jobs to
-accurately cover the original data.
+This limitation has since been alleviated.  As of 6.4.0, it is now considered best practice to combine all rollup configurations
+into a single job.
 
 As an example, if your index has two types of documents:
 
@@ -157,7 +156,7 @@ and
 --------------------------------------------------
 // NOTCONSOLE
 
-it may be tempting to create a single, combined rollup job which covers both of these document types, something like this:
+the best practice is to combine them into a single rollup job which covers both of these document types, like this:
 
 [source,js]
 --------------------------------------------------
@@ -191,222 +190,10 @@ PUT _xpack/rollup/job/combined
 --------------------------------------------------
 // NOTCONSOLE
 
-You can see that it includes a `terms` grouping on both "node" and "title", fields that are mutually exclusive in the document types.
-*This will not work.*  Because the `composite` aggregation (and by extension, Rollup) only returns buckets when all keys are non-null,
-and there are no documents that have both a "node" field and a "title" field, this rollup job will not produce any rollups.
-
-Instead, you should configure two independent jobs (sharing the same index, or going to separate indices):
-
-[source,js]
---------------------------------------------------
-PUT _xpack/rollup/job/sensor
-{
-    "index_pattern": "data-*",
-    "rollup_index": "data_rollup",
-    "cron": "*/30 * * * * ?",
-    "page_size" :1000,
-    "groups" : {
-      "date_histogram": {
-        "field": "timestamp",
-        "interval": "1h",
-        "delay": "7d"
-      },
-      "terms": {
-        "fields": ["node"]
-      }
-    },
-    "metrics": [
-        {
-            "field": "temperature",
-            "metrics": ["min", "max", "sum"]
-        }
-    ]
-}
---------------------------------------------------
-// NOTCONSOLE
-
-[source,js]
---------------------------------------------------
-PUT _xpack/rollup/job/purchases
-{
-    "index_pattern": "data-*",
-    "rollup_index": "data_rollup",
-    "cron": "*/30 * * * * ?",
-    "page_size" :1000,
-    "groups" : {
-      "date_histogram": {
-        "field": "timestamp",
-        "interval": "1h",
-        "delay": "7d"
-      },
-      "terms": {
-        "fields": ["title"]
-      }
-    },
-    "metrics": [
-        {
-            "field": "price",
-            "metrics": ["avg"]
-        }
-    ]
-}
---------------------------------------------------
-// NOTCONSOLE
-
-Notice that each job now deals with a single "document type", and will not run into the limitations described above.  We are working on changes
-in core Elasticsearch to remove this limitation from the `composite` aggregation, and the documentation will be updated accordingly
-when this particular scenario is fixed.
-
 === Doc counts and overlapping jobs
 
-There is an issue with doc counts, related to the above grouping limitation.  Imagine you have two Rollup jobs saving to the same index, where
-one job is a "subset" of another job.
+There was previously an issue with document counts on "overlapping" job configurations, driven by the same internal implementation detail.
+If there were  two Rollup jobs saving to the same index, where one job is a "subset" of another job, it was possible that document counts
+could be incorrect for certain aggregation arrangements.
 
-For example, you might have jobs with these two groupings:
-
-[source,js]
---------------------------------------------------
-PUT _xpack/rollup/job/sensor-all
-{
-    "groups" : {
-      "date_histogram": {
-        "field": "timestamp",
-        "interval": "1h",
-        "delay": "7d"
-      },
-      "terms": {
-        "fields": ["node"]
-      }
-    },
-    "metrics": [
-        {
-            "field": "price",
-            "metrics": ["avg"]
-        }
-    ]
-    ...
-}
---------------------------------------------------
-// NOTCONSOLE
-
-and
-
-[source,js]
---------------------------------------------------
-PUT _xpack/rollup/job/sensor-building
-{
-    "groups" : {
-      "date_histogram": {
-        "field": "timestamp",
-        "interval": "1h",
-        "delay": "7d"
-      },
-      "terms": {
-        "fields": ["node", "building"]
-      }
-    }
-    ...
-}
---------------------------------------------------
-// NOTCONSOLE
-
-
-The first job `sensor-all` contains the groupings and metrics that apply to all data in the index.  The second job is rolling up a subset
-of data (in different buildings) which also include a building identifier.  You did this because combining them would run into the limitation
-described in the previous section.
-
-This _mostly_ works, but can sometimes return incorrect `doc_counts` when you search.  All metrics will be valid however.
-
-The issue arises from the composite agg limitation described before, combined with search-time optimization.  Imagine you try to run the
-following aggregation:
-
-[source,js]
---------------------------------------------------
-"aggs" : {
-  "nodes": {
-    "terms": {
-      "field": "node"
-    }
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-This aggregation could be serviced by either `sensor-all` or `sensor-building` job, since they both group on the node field.  So the RollupSearch
-API will search both of them and merge results.  This will result in *correct* doc_counts and *correct* metrics.  No problem here.
-
-The issue arises from an aggregation that can _only_ be serviced by `sensor-building`, like this one:
-
-[source,js]
---------------------------------------------------
-"aggs" : {
-  "nodes": {
-    "terms": {
-      "field": "node"
-    },
-    "aggs": {
-      "building": {
-        "terms": {
-          "field": "building"
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-Now we run into a problem.  The RollupSearch API will correctly identify that only `sensor-building` job has all the required components
-to answer the aggregation, and will search it exclusively.  Unfortunately, due to the composite aggregation limitation, that job only
-rolled up documents that have both a "node" and a "building" field.  Meaning that the doc_counts for the `"nodes"` aggregation will not
-include counts for any document that doesn't have `[node, building]` fields.
-
-- The `doc_count` for `"nodes"` aggregation will be incorrect because it only contains counts for `nodes` that also have buildings
-- The `doc_count` for `"buildings"` aggregation will be correct
-- Any metrics, on any level, will be correct
-
-==== Workarounds
-
-There are two main workarounds if you find yourself with a schema like the above.
-
-Easiest and most robust method: use separate indices to store your rollups.  The limitations arise because you have several document
-schemas co-habitating in a single index, which makes it difficult for rollups to correctly summarize.  If you make several rollup
-jobs and store them in separate indices, these sorts of difficulties do not arise.  It does, however, keep you from searching across several
-different rollup indices at the same time.
-
-The other workaround is to include an "off-target" aggregation in the query, which pulls in the "superset" job and corrects the doc counts.
-The RollupSearch API determines the best job to search for each "leaf node" in the aggregation tree.  So if we include a metric agg on `price`,
-which was only defined in the `sensor-all` job, that will "pull in" the other job:
-
-[source,js]
---------------------------------------------------
-"aggs" : {
-  "nodes": {
-    "terms": {
-      "field": "node"
-    },
-    "aggs": {
-      "building": {
-        "terms": {
-          "field": "building"
-        }
-      },
-      "avg_price": {
-        "avg": { "field": "price" } <1>
-      }
-    }
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-<1> Adding an avg aggregation here will fix the doc counts
-
-Because only `sensor-all` job had an `avg` on the price field, the RollupSearch API is forced to pull in that additional job for searching,
-and will merge/correct the doc_counts as appropriate.  This sort of workaround applies to any additional aggregation -- metric or bucketing --
-although it can be tedious to look through the jobs and determine the right one to add.
-
-==== Status
-
-We realize this is an onerous limitation, and somewhat breaks the rollup contract of "pick the fields to rollup, we do the rest".  We are
-actively working to get the limitation to `composite` agg fixed, and the related issues in Rollup.  The documentation will be updated when
-the fix is implemented.
\ No newline at end of file
+This issue has also since been eliminated in 6.4.0.
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfig.java
index 4b4e4cf7b7c..4a9fbde61d6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistoGroupConfig.java
@@ -159,7 +159,6 @@ public class DateHistoGroupConfig implements Writeable, ToXContentFragment {
         vsBuilder.dateHistogramInterval(interval);
         vsBuilder.field(field);
         vsBuilder.timeZone(timeZone);
-
         return Collections.singletonList(vsBuilder);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java
index 8b8d53b4ce9..2b1511077d9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java
@@ -96,6 +96,7 @@ public class HistoGroupConfig implements Writeable, ToXContentFragment {
                     = new HistogramValuesSourceBuilder(RollupField.formatIndexerAggName(f, HistogramAggregationBuilder.NAME));
             vsBuilder.interval(interval);
             vsBuilder.field(f);
+            vsBuilder.missingBucket(true);
             return vsBuilder;
         }).collect(Collectors.toList());
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
index 2f1c35a73ed..da73020f008 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
@@ -80,6 +80,7 @@ public class TermsGroupConfig implements Writeable, ToXContentFragment {
             TermsValuesSourceBuilder vsBuilder
                     = new TermsValuesSourceBuilder(RollupField.formatIndexerAggName(f, TermsAggregationBuilder.NAME));
             vsBuilder.field(f);
+            vsBuilder.missingBucket(true);
             return vsBuilder;
         }).collect(Collectors.toList());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
index 9938f3a4196..ae171f138cf 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.core.rollup;
 
 import org.apache.http.HttpStatus;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -27,21 +26,13 @@ import static org.junit.Assert.assertEquals;
 
 public class RollupRestTestStateCleaner {
 
-    private final Logger logger;
-    private final RestClient adminClient;
-
-    public RollupRestTestStateCleaner(Logger logger, RestClient adminClient) {
-        this.logger = logger;
-        this.adminClient = adminClient;
-    }
-
-    public void clearRollupMetadata() throws Exception {
-        deleteAllJobs();
-        waitForPendingTasks();
+    public static void clearRollupMetadata(RestClient adminClient) throws Exception {
+        deleteAllJobs(adminClient);
+        waitForPendingTasks(adminClient);
         // indices will be deleted by the ESRestTestCase class
     }
 
-    private void waitForPendingTasks() throws Exception {
+    private static void waitForPendingTasks(RestClient adminClient) throws Exception {
         ESTestCase.assertBusy(() -> {
             try {
                 Response response = adminClient.performRequest("GET", "/_cat/tasks",
@@ -71,7 +62,7 @@ public class RollupRestTestStateCleaner {
     }
 
     @SuppressWarnings("unchecked")
-    private void deleteAllJobs() throws Exception {
+    private static void deleteAllJobs(RestClient adminClient) throws Exception {
         Response response = adminClient.performRequest("GET", "/_xpack/rollup/job/_all");
         Map<String, Object> jobs = ESRestTestCase.entityAsMap(response);
         @SuppressWarnings("unchecked")
@@ -83,9 +74,7 @@ public class RollupRestTestStateCleaner {
         }
 
         for (Map<String, Object> jobConfig : jobConfigs) {
-            logger.debug(jobConfig);
             String jobId = (String) ((Map<String, Object>) jobConfig.get("config")).get("id");
-            logger.debug("Deleting job " + jobId);
             try {
                 response = adminClient.performRequest("DELETE", "/_xpack/rollup/job/" + jobId);
             } catch (Exception e) {
@@ -95,7 +84,8 @@ public class RollupRestTestStateCleaner {
     }
 
     private static String responseEntityToString(Response response) throws Exception {
-        try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(),
+            StandardCharsets.UTF_8))) {
             return reader.lines().collect(Collectors.joining("\n"));
         }
     }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
index ba1002896c0..4042e98ef93 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
@@ -382,6 +382,7 @@ public class RollupResponseTranslator {
             });
         } else if (rolled instanceof StringTerms) {
             return unrollMultiBucket(rolled, original, currentTree, (bucket, bucketCount, subAggs) -> {
+
                 BytesRef key = new BytesRef(bucket.getKeyAsString().getBytes(StandardCharsets.UTF_8));
                 assert bucketCount >= 0;
                 //TODO expose getFormatter(), keyed upstream in Core
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
index 081b97b4ee7..889dfa3ac8e 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
@@ -173,6 +173,14 @@ public class TransportPutRollupJobAction extends TransportMasterNodeAction<PutRo
             }
 
             Map<String, Object> rollupMeta = (Map<String, Object>)((Map<String, Object>) m).get(RollupField.ROLLUP_META);
+
+            String stringVersion = (String)((Map<String, Object>) m).get(Rollup.ROLLUP_TEMPLATE_VERSION_FIELD);
+            if (stringVersion == null) {
+                listener.onFailure(new IllegalStateException("Could not determine version of existing rollup metadata for index ["
+                    + indexName + "]"));
+                return;
+            }
+
             if (rollupMeta.get(job.getConfig().getId()) != null) {
                 String msg = "Cannot create rollup job [" + job.getConfig().getId()
                         + "] because job was previously created (existing metadata).";
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
index e180e34c4cc..efac4c2d61b 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
@@ -45,7 +45,7 @@ class IndexerUtils {
      * @param rollupIndex  The index that holds rollups for this job
      * @return             A list of rolled documents derived from the response
      */
-    static List<IndexRequest> processBuckets(CompositeAggregation agg, String rollupIndex, RollupJobStats stats, 
+    static List<IndexRequest> processBuckets(CompositeAggregation agg, String rollupIndex, RollupJobStats stats,
                                              GroupConfig groupConfig, String jobId) {
 
         logger.debug("Buckets: [" + agg.getBuckets().size() + "][" + jobId + "]");
@@ -80,6 +80,7 @@ class IndexerUtils {
             doc.put(k + "." + RollupField.COUNT_FIELD, count);
 
             if (k.endsWith("." + DateHistogramAggregationBuilder.NAME)) {
+                assert v != null;
                 doc.put(k + "." + RollupField.TIMESTAMP, v);
                 doc.put(k  + "." + RollupField.INTERVAL, groupConfig.getDateHisto().getInterval());
                 doc.put(k  + "." + DateHistoGroupConfig.TIME_ZONE, groupConfig.getDateHisto().getTimeZone().toString());
@@ -87,10 +88,18 @@ class IndexerUtils {
             } else if (k.endsWith("." + HistogramAggregationBuilder.NAME)) {
                 doc.put(k + "." + RollupField.VALUE, v);
                 doc.put(k + "." + RollupField.INTERVAL, groupConfig.getHisto().getInterval());
-                docID.update(Numbers.doubleToBytes((Double)v), 0, 8);
+                if (v == null) {
+                    // Arbitrary value to update the doc ID with for nulls
+                    docID.update(19);
+                } else {
+                    docID.update(Numbers.doubleToBytes((Double) v), 0, 8);
+                }
             } else if (k.endsWith("." + TermsAggregationBuilder.NAME)) {
                 doc.put(k + "." + RollupField.VALUE, v);
-                if (v instanceof String) {
+                if (v == null) {
+                    // Arbitrary value to update the doc ID with for nulls
+                    docID.update(19);
+                } else if (v instanceof String) {
                     byte[] vs = ((String) v).getBytes(StandardCharsets.UTF_8);
                     docID.update(vs, 0, vs.length);
                 } else if (v instanceof Long) {
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
index a07f1e7d32e..1711c0e34eb 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
@@ -401,6 +401,7 @@ public abstract class RollupIndexer {
             composite.setMetaData(metadata);
         }
         composite.size(config.getPageSize());
+
         return composite;
     }
 
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
index 7b03d8e8d03..98e3ad8197a 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
@@ -1082,6 +1082,54 @@ public class RollupResponseTranslationTests extends AggregatorTestCase {
         assertThat(unrolled.toString(), not(equalTo(responses.get(1).toString())));
     }
 
+    public void testStringTermsNullValue() throws IOException {
+        TermsAggregationBuilder nonRollupTerms = new TermsAggregationBuilder("terms", ValueType.STRING)
+            .field("stringField");
+
+        TermsAggregationBuilder rollupTerms = new TermsAggregationBuilder("terms", ValueType.STRING)
+            .field("stringfield.terms." + RollupField.VALUE)
+            .subAggregation(new SumAggregationBuilder("terms." + COUNT_FIELD)
+                .field("stringfield.terms." + RollupField.COUNT_FIELD));
+
+        KeywordFieldMapper.Builder nrBuilder = new KeywordFieldMapper.Builder("terms");
+        KeywordFieldMapper.KeywordFieldType nrFTterm = nrBuilder.fieldType();
+        nrFTterm.setHasDocValues(true);
+        nrFTterm.setName(nonRollupTerms.field());
+
+        KeywordFieldMapper.Builder rBuilder = new KeywordFieldMapper.Builder("terms");
+        KeywordFieldMapper.KeywordFieldType rFTterm = rBuilder.fieldType();
+        rFTterm.setHasDocValues(true);
+        rFTterm.setName(rollupTerms.field());
+
+        NumberFieldMapper.Builder valueBuilder = new NumberFieldMapper.Builder("terms." + RollupField.COUNT_FIELD,
+            NumberFieldMapper.NumberType.LONG);
+        MappedFieldType rFTvalue = valueBuilder.fieldType();
+        rFTvalue.setHasDocValues(true);
+        rFTvalue.setName("stringfield.terms." + RollupField.COUNT_FIELD);
+
+        List<InternalAggregation> responses = doQueries(new MatchAllDocsQuery(),
+            iw -> {
+                iw.addDocument(stringValueDoc("abc"));
+                iw.addDocument(stringValueDoc("abc"));
+                iw.addDocument(stringValueDoc("abc"));
+
+                // off target
+                Document doc = new Document();
+                doc.add(new SortedSetDocValuesField("otherField", new BytesRef("other")));
+                iw.addDocument(doc);
+            }, nonRollupTerms,
+            iw -> {
+                iw.addDocument(stringValueRollupDoc("abc", 3));
+            }, rollupTerms,
+            new MappedFieldType[]{nrFTterm}, new MappedFieldType[]{rFTterm, rFTvalue});
+
+        InternalAggregation unrolled = RollupResponseTranslator.unrollAgg(responses.get(1), null, null, 0);
+
+        // The null_value placeholder should be removed from the response and not visible here
+        assertThat(unrolled.toString(), equalTo(responses.get(0).toString()));
+        assertThat(unrolled.toString(), not(equalTo(responses.get(1).toString())));
+    }
+
     public void testLongTerms() throws IOException {
         TermsAggregationBuilder nonRollupTerms = new TermsAggregationBuilder("terms", ValueType.LONG)
                 .field("longField");
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
index 64cf9d2e3fe..58fa9d4533b 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
@@ -28,9 +28,12 @@ import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
 import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
+import org.elasticsearch.xpack.rollup.Rollup;
 import org.mockito.ArgumentCaptor;
 
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -203,6 +206,43 @@ public class PutJobStateMachineTests extends ESTestCase {
         verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any());
     }
 
+    @SuppressWarnings("unchecked")
+    public void testNoMappingVersion() {
+        RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
+
+        ActionListener<PutRollupJobAction.Response> testListener = ActionListener.wrap(response -> {
+            fail("Listener success should not have been triggered.");
+        }, e -> {
+            assertThat(e.getMessage(), equalTo("Could not determine version of existing rollup metadata for index ["
+                + job.getConfig().getRollupIndex() + "]"));
+        });
+
+        Logger logger = mock(Logger.class);
+        Client client = mock(Client.class);
+
+        ArgumentCaptor<ActionListener> requestCaptor = ArgumentCaptor.forClass(ActionListener.class);
+        doAnswer(invocation -> {
+            GetMappingsResponse response = mock(GetMappingsResponse.class);
+            Map<String, Object> m = new HashMap<>(2);
+            m.put(RollupField.ROLLUP_META,
+                Collections.singletonMap(job.getConfig().getId(), job.getConfig()));
+            MappingMetaData meta = new MappingMetaData(RollupField.TYPE_NAME,
+                Collections.singletonMap("_meta", m));
+            ImmutableOpenMap.Builder<String, MappingMetaData> builder = ImmutableOpenMap.builder(1);
+            builder.put(RollupField.TYPE_NAME, meta);
+
+            ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> builder2 = ImmutableOpenMap.builder(1);
+            builder2.put(job.getConfig().getRollupIndex(), builder.build());
+
+            when(response.getMappings()).thenReturn(builder2.build());
+            requestCaptor.getValue().onResponse(response);
+            return null;
+        }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture());
+
+        TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger);
+        verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any());
+    }
+
     @SuppressWarnings("unchecked")
     public void testJobAlreadyInMapping() {
         RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap());
@@ -219,10 +259,12 @@ public class PutJobStateMachineTests extends ESTestCase {
         ArgumentCaptor<ActionListener> requestCaptor = ArgumentCaptor.forClass(ActionListener.class);
         doAnswer(invocation -> {
             GetMappingsResponse response = mock(GetMappingsResponse.class);
+            Map<String, Object> m = new HashMap<>(2);
+            m.put(Rollup.ROLLUP_TEMPLATE_VERSION_FIELD, Version.V_6_4_0);
+            m.put(RollupField.ROLLUP_META,
+                Collections.singletonMap(job.getConfig().getId(), job.getConfig()));
             MappingMetaData meta = new MappingMetaData(RollupField.TYPE_NAME,
-                    Collections.singletonMap("_meta",
-                            Collections.singletonMap(RollupField.ROLLUP_META,
-                                    Collections.singletonMap(job.getConfig().getId(), job.getConfig()))));
+                Collections.singletonMap("_meta", m));
             ImmutableOpenMap.Builder<String, MappingMetaData> builder = ImmutableOpenMap.builder(1);
             builder.put(RollupField.TYPE_NAME, meta);
 
@@ -258,9 +300,12 @@ public class PutJobStateMachineTests extends ESTestCase {
         ArgumentCaptor<ActionListener> requestCaptor = ArgumentCaptor.forClass(ActionListener.class);
         doAnswer(invocation -> {
             GetMappingsResponse response = mock(GetMappingsResponse.class);
+            Map<String, Object> m = new HashMap<>(2);
+            m.put(Rollup.ROLLUP_TEMPLATE_VERSION_FIELD, Version.V_6_4_0);
+            m.put(RollupField.ROLLUP_META,
+                Collections.singletonMap(unrelatedJob.getId(), unrelatedJob));
             MappingMetaData meta = new MappingMetaData(RollupField.TYPE_NAME,
-                    Collections.singletonMap("_meta", Collections.singletonMap(RollupField.ROLLUP_META,
-                                    Collections.singletonMap(unrelatedJob.getId(), unrelatedJob))));
+                Collections.singletonMap("_meta", m));
             ImmutableOpenMap.Builder<String, MappingMetaData> builder = ImmutableOpenMap.builder(1);
             builder.put(RollupField.TYPE_NAME, meta);
 
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
index 53421faa9bc..07ad0af7f1c 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
@@ -15,6 +15,7 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -33,12 +34,13 @@ import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggre
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
+import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobStats;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
+import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 import org.joda.time.DateTime;
 import org.mockito.stubbing.Answer;
 
@@ -50,8 +52,8 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.mockito.Mockito.mock;
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class IndexerUtilsTests extends AggregatorTestCase {
@@ -359,6 +361,106 @@ public class IndexerUtilsTests extends AggregatorTestCase {
         assertThat(docs.get(0).id(), equalTo("1237859798"));
     }
 
+    public void testNullKeys() {
+        CompositeAggregation composite = mock(CompositeAggregation.class);
+
+        when(composite.getBuckets()).thenAnswer((Answer<List<CompositeAggregation.Bucket>>) invocationOnMock -> {
+            List<CompositeAggregation.Bucket> foos = new ArrayList<>();
+
+            CompositeAggregation.Bucket bucket = mock(CompositeAggregation.Bucket.class);
+            LinkedHashMap<String, Object> keys = new LinkedHashMap<>(3);
+            keys.put("bar.terms", null);
+            keys.put("abc.histogram", null);
+            when(bucket.getKey()).thenReturn(keys);
+
+            Aggregations aggs = new Aggregations(Collections.emptyList());
+            when(bucket.getAggregations()).thenReturn(aggs);
+            when(bucket.getDocCount()).thenReturn(1L);
+
+            foos.add(bucket);
+
+            return foos;
+        });
+
+        GroupConfig.Builder groupConfig = ConfigTestHelpers.getGroupConfig();
+        groupConfig.setHisto(ConfigTestHelpers.getHisto().setFields(Collections.singletonList("abc")).build());
+
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, "foo", new RollupJobStats(), groupConfig.build(), "foo");
+        assertThat(docs.size(), equalTo(1));
+        assertFalse(Strings.isNullOrEmpty(docs.get(0).id()));
+    }
+
+    public void testMissingBuckets() throws IOException {
+        String indexName = randomAlphaOfLengthBetween(1, 10);
+        RollupJobStats stats= new RollupJobStats(0, 0, 0, 0);
+
+        String metricField = "metric_field";
+        String valueField = "value_field";
+
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+
+        int numDocs = 10;
+
+        for (int i = 0; i < numDocs; i++) {
+            Document document = new Document();
+
+            // Every other doc omit the valueField, so that we get some null buckets
+            if (i % 2 == 0) {
+                document.add(new SortedNumericDocValuesField(valueField, i));
+                document.add(new LongPoint(valueField, i));
+            }
+            document.add(new SortedNumericDocValuesField(metricField, i));
+            document.add(new LongPoint(metricField, i));
+            indexWriter.addDocument(document);
+        }
+
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+
+        MappedFieldType valueFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        valueFieldType.setName(valueField);
+        valueFieldType.setHasDocValues(true);
+        valueFieldType.setName(valueField);
+
+        MappedFieldType metricFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        metricFieldType.setName(metricField);
+        metricFieldType.setHasDocValues(true);
+        metricFieldType.setName(metricField);
+
+        // Setup the composite agg
+        TermsGroupConfig termsGroupConfig = new TermsGroupConfig.Builder().setFields(Collections.singletonList(valueField)).build();
+        CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME,
+            termsGroupConfig.toBuilders()).size(numDocs*2);
+
+        MetricConfig metricConfig = new MetricConfig.Builder().setField(metricField).setMetrics(Collections.singletonList("max")).build();
+        metricConfig.toBuilders().forEach(compositeBuilder::subAggregation);
+
+        Aggregator aggregator = createAggregator(compositeBuilder, indexSearcher, valueFieldType, metricFieldType);
+        aggregator.preCollection();
+        indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+        aggregator.postCollection();
+        CompositeAggregation composite = (CompositeAggregation) aggregator.buildAggregation(0L);
+        indexReader.close();
+        directory.close();
+
+        List<IndexRequest> docs = IndexerUtils.processBuckets(composite, indexName, stats,
+            ConfigTestHelpers.getGroupConfig().build(), "foo");
+
+        assertThat(docs.size(), equalTo(6));
+        for (IndexRequest doc : docs) {
+            Map<String, Object> map = doc.sourceAsMap();
+            Object value = map.get(valueField + "." + TermsAggregationBuilder.NAME + "." + RollupField.VALUE);
+            if (value == null) {
+                assertThat(map.get(valueField + "." + TermsAggregationBuilder.NAME + "." + RollupField.COUNT_FIELD), equalTo(5));
+            } else {
+                assertThat(map.get(valueField + "." + TermsAggregationBuilder.NAME + "." + RollupField.COUNT_FIELD), equalTo(1));
+            }
+        }
+    }
+
     interface Mock {
         List<? extends CompositeAggregation.Bucket> getBuckets();
     }
diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
index 412c75f0e63..f1d9eb1fb3f 100644
--- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
+++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
@@ -263,7 +263,7 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase {
      */
     private void clearRollupState() throws Exception {
         if (isRollupTest()) {
-            new RollupRestTestStateCleaner(logger, adminClient()).clearRollupMetadata();
+            RollupRestTestStateCleaner.clearRollupMetadata(adminClient());
         }
     }
 
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index 5276abdbfb1..ba6f9e91678 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -18,6 +18,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.StreamsUtils;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -531,7 +532,10 @@ public class FullClusterRestartIT extends ESRestTestCase {
         // check that the rollup job is started using the RollUp API
         final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/" + rollupJob);
         Map<String, Object> getRollupJobResponse = toMap(client().performRequest(getRollupJobRequest));
-        assertThat(ObjectPath.eval("jobs.0.status.job_state", getRollupJobResponse), expectedStates);
+        Map<String, Object> job = getJob(getRollupJobResponse, rollupJob);
+        if (job != null) {
+            assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
+        }
 
         // check that the rollup job is started using the Tasks API
         final Request taskRequest = new Request("GET", "_tasks");
@@ -547,15 +551,27 @@ public class FullClusterRestartIT extends ESRestTestCase {
         // check that the rollup job is started using the Cluster State API
         final Request clusterStateRequest = new Request("GET", "_cluster/state/metadata");
         Map<String, Object> clusterStateResponse = toMap(client().performRequest(clusterStateRequest));
-        Map<String, Object> rollupJobTask = ObjectPath.eval("metadata.persistent_tasks.tasks.0", clusterStateResponse);
-        assertThat(ObjectPath.eval("id", rollupJobTask), equalTo("rollup-job-test"));
+        List<Map<String, Object>> rollupJobTasks = ObjectPath.eval("metadata.persistent_tasks.tasks", clusterStateResponse);
 
-        // Persistent task state field has been renamed in 6.4.0 from "status" to "state"
-        final String stateFieldName = (runningAgainstOldCluster && oldClusterVersion.before(Version.V_6_4_0)) ? "status" : "state";
+        boolean hasRollupTask = false;
+        for (Map<String, Object> task : rollupJobTasks) {
+            if (ObjectPath.eval("id", task).equals(rollupJob)) {
+                hasRollupTask = true;
+
+                // Persistent task state field has been renamed in 6.4.0 from "status" to "state"
+                final String stateFieldName
+                    = (runningAgainstOldCluster && oldClusterVersion.before(Version.V_6_4_0)) ? "status" : "state";
+
+                final String jobStateField = "task.xpack/rollup/job." + stateFieldName + ".job_state";
+                assertThat("Expected field [" + jobStateField + "] to be started or indexing in " + task.get("id"),
+                    ObjectPath.eval(jobStateField, task), expectedStates);
+                break;
+            }
+        }
+        if (hasRollupTask == false) {
+            fail("Expected persistent task for [" + rollupJob + "] but none found.");
+        }
 
-        final String jobStateField = "task.xpack/rollup/job." + stateFieldName + ".job_state";
-        assertThat("Expected field [" + jobStateField + "] to be started or indexing in " + rollupJobTask,
-            ObjectPath.eval(jobStateField, rollupJobTask), expectedStates);
     }
 
     private void waitForRollUpJob(final String rollupJob, final Matcher<?> expectedStates) throws Exception {
@@ -563,7 +579,34 @@ public class FullClusterRestartIT extends ESRestTestCase {
             final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/" + rollupJob);
             Response getRollupJobResponse = client().performRequest(getRollupJobRequest);
             assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
-            assertThat(ObjectPath.eval("jobs.0.status.job_state", toMap(getRollupJobResponse)), expectedStates);
+
+            Map<String, Object> job = getJob(getRollupJobResponse, rollupJob);
+            if (job != null) {
+                assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
+            }
         }, 30L, TimeUnit.SECONDS);
     }
+
+    private Map<String, Object> getJob(Response response, String targetJobId) throws IOException {
+        return getJob(ESRestTestCase.entityAsMap(response), targetJobId);
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> getJob(Map<String, Object> jobsMap, String targetJobId) throws IOException {
+
+        List<Map<String, Object>> jobs =
+            (List<Map<String, Object>>) XContentMapValues.extractValue("jobs", jobsMap);
+
+        if (jobs == null) {
+            return null;
+        }
+
+        for (Map<String, Object> job : jobs) {
+            String jobId = (String) ((Map<String, Object>) job.get("config")).get("id");
+            if (jobId.equals(targetJobId)) {
+                return job;
+            }
+        }
+        return null;
+    }
 }

From 1bf205f862a32640d4febcd61cea49a04e406fa0 Mon Sep 17 00:00:00 2001
From: Michael Basnight <mbasnight@gmail.com>
Date: Fri, 13 Jul 2018 09:33:29 -0500
Subject: [PATCH 006/260] Test: Fix a second case of bad watch creation

There was still a case with a null text that allowed for 0 attachments
to be created. This commit ensures that greater than zero are created
if the text is null. Otherwise, it uses the same logic to create 0 to 3
random attachments.

Closes #31948
---
 .../notification/slack/message/SlackMessageTests.java        | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java
index 14d732064e5..10544e464ac 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java
@@ -461,7 +461,6 @@ public class SlackMessageTests extends ESTestCase {
         assertThat(parsed, equalTo(template));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31948")
     public void testTemplateRender() throws Exception {
         Settings settings = SlackMessageDefaultsTests.randomSettings();
         SlackMessageDefaults defaults = new SlackMessageDefaults(settings);
@@ -483,7 +482,9 @@ public class SlackMessageTests extends ESTestCase {
             templateBuilder.setText(randomAlphaOfLength(10));
         }
         if (templateBuilder.text == null || randomBoolean()) {
-            int count = randomIntBetween(0, 3);
+            // ensure at least one attachment in the event the text is null
+            int minimumAttachments = templateBuilder.text == null ? 1 : 0;
+            int count = randomIntBetween(minimumAttachments, 3);
             for (int i = 0; i < count; i++) {
                 Attachment.Template.Builder attachmentBuilder = createRandomAttachmentTemplateBuilder();
                 templateBuilder.addAttachments(attachmentBuilder);

From f9791cf158ad6403b2872897d45ff7c773269569 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@gmail.com>
Date: Fri, 13 Jul 2018 15:49:26 +0100
Subject: [PATCH 007/260] Remove deprecated
 AnalysisPlugin#requriesAnalysisSettings method (#32037)

---
 .../java/org/elasticsearch/plugins/AnalysisPlugin.java | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java b/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
index 27b5667d3bb..c85981f8dcb 100644
--- a/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
@@ -128,16 +128,6 @@ public interface AnalysisPlugin {
         return emptyMap();
     }
 
-    /**
-     * Mark an {@link AnalysisProvider} as requiring the index's settings.
-     *
-     * @deprecated use {@link #requiresAnalysisSettings(AnalysisProvider)}
-     */
-    @Deprecated
-    static <T> AnalysisProvider<T> requriesAnalysisSettings(AnalysisProvider<T> provider) {
-        return requiresAnalysisSettings(provider);
-    }
-
     /**
      * Mark an {@link AnalysisProvider} as requiring the index's settings.
      */

From 80492cacfcadbb0ea513914e1012a725f8739b55 Mon Sep 17 00:00:00 2001
From: Mayya Sharipova <mayya.sharipova@elastic.co>
Date: Fri, 13 Jul 2018 11:40:03 -0400
Subject: [PATCH 008/260] Add second level of field collapsing (#31808)

* Put second level collapse under inner_hits

Closes #24855
---
 .../search/request/collapse.asciidoc          | 102 +++++++++++++
 .../search/115_multiple_field_collapsing.yml  | 141 ++++++++++++++++++
 .../action/search/ExpandSearchPhase.java      |   8 +-
 .../index/query/InnerHitBuilder.java          |  49 +++++-
 4 files changed, 296 insertions(+), 4 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml

diff --git a/docs/reference/search/request/collapse.asciidoc b/docs/reference/search/request/collapse.asciidoc
index 97d85329330..192495e5d6d 100644
--- a/docs/reference/search/request/collapse.asciidoc
+++ b/docs/reference/search/request/collapse.asciidoc
@@ -116,3 +116,105 @@ The default is based on the number of data nodes and the default search thread p
 
 WARNING: `collapse` cannot be used in conjunction with <<search-request-scroll, scroll>>,
 <<search-request-rescore, rescore>> or <<search-request-search-after, search after>>.
+
+==== Second level of collapsing
+
+Second level of collapsing is also supported and is applied to `inner_hits`.
+For example, the following request finds the top scored tweets for
+each country, and within each country finds the top scored tweets
+for each user.
+
+[source,js]
+--------------------------------------------------
+GET /twitter/_search
+{
+    "query": {
+        "match": {
+            "message": "elasticsearch"
+        }
+    },
+    "collapse" : {
+        "field" : "country",
+        "inner_hits" : {
+            "name": "by_location",
+            "collapse" : {"field" : "user"},
+            "size": 3
+        }
+    }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+
+Response:
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "hits": [
+        {
+            "_index": "twitter",
+            "_type": "_doc",
+            "_id": "9",
+            "_score": ...,
+            "_source": {...},
+            "fields": {"country": ["UK"]},
+            "inner_hits":{
+                "by_location": {
+                    "hits": {
+                       ...,
+                       "hits": [
+                          {
+                            ...
+                            "fields": {"user" : ["user124"]}
+                          },
+                          {
+                            ...
+                            "fields": {"user" : ["user589"]}
+                          },
+                          {
+                            ...
+                             "fields": {"user" : ["user001"]}
+                          }
+                       ]
+                    }
+                 }
+            }
+        },
+        {
+            "_index": "twitter",
+            "_type": "_doc",
+            "_id": "1",
+            "_score": ..,
+            "_source": {...},
+            "fields": {"country": ["Canada"]},
+            "inner_hits":{
+                "by_location": {
+                    "hits": {
+                       ...,
+                       "hits": [
+                          {
+                            ...
+                            "fields": {"user" : ["user444"]}
+                          },
+                          {
+                            ...
+                            "fields": {"user" : ["user1111"]}
+                          },
+                          {
+                            ...
+                             "fields": {"user" : ["user999"]}
+                          }
+                       ]
+                    }
+                 }
+            }
+
+        },
+        ....
+    ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+NOTE: Second level of of collapsing doesn't allow `inner_hits`.
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml
new file mode 100644
index 00000000000..212ce6785a1
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/115_multiple_field_collapsing.yml
@@ -0,0 +1,141 @@
+---
+"two levels fields collapsing":
+    - skip:
+        version: " - 6.99.99"
+        reason: using multiple field collapsing from 7.0 on
+    - do:
+        indices.create:
+          index: addresses
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+            mappings:
+              _doc:
+                properties:
+                  country: {"type": "keyword"}
+                  city: {"type": "keyword"}
+                  address: {"type": "text"}
+
+    - do:
+        bulk:
+          refresh: true
+          body:
+            - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "1" } }'
+            - '{"country" : "Canada", "city" : "Saskatoon", "address" : "701 Victoria Avenue" }'
+            - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "2" } }'
+            - '{"country" : "Canada", "city" : "Toronto", "address" : "74 Victoria Street, Suite, 74 Victoria Street, Suite 300" }'
+            - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "3" } }'
+            - '{"country" : "Canada", "city" : "Toronto", "address" : "350 Victoria St" }'
+            - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "4" } }'
+            - '{"country" : "Canada", "city" : "Toronto", "address" : "20 Victoria Street" }'
+            - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "5" } }'
+            - '{"country" : "UK", "city" : "London", "address" : "58 Victoria Street" }'
+            - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "6" } }'
+            - '{"country" : "UK", "city" : "London", "address" : "Victoria Street Victoria Palace Theatre" }'
+            - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "7" } }'
+            - '{"country" : "UK", "city" : "Manchester", "address" : "75 Victoria street Westminster" }'
+            - '{ "index" : { "_index" : "addresses", "_type" : "_doc", "_id" : "8" } }'
+            - '{"country" : "UK", "city" : "London", "address" : "Victoria Station Victoria Arcade" }'
+
+
+    # ************* error if internal collapse contains inner_hits
+    - do:
+        catch:  /parse_exception/
+        search:
+          index: addresses
+          body:
+            query: { "match" : { "address" : "victoria" }}
+            collapse:
+              field: country
+              inner_hits:
+                collapse:
+                  field : city
+                  inner_hits: {}
+
+
+    # ************* error if internal collapse contains another collapse
+    - do:
+        catch:  /parse_exception/
+        search:
+          index: addresses
+          body:
+            query: { "match" : { "address" : "victoria" }}
+            collapse:
+              field: country
+              inner_hits:
+                collapse:
+                  field : city
+                  collapse:  { field: city }
+
+
+
+    # ************* top scored
+    - do:
+        search:
+          index: addresses
+          body:
+            query: { "match" : { "address" : "victoria" }}
+            collapse:
+              field: country
+              inner_hits:
+                name: by_location
+                size: 3
+                collapse:
+                  field : city
+
+    - match: { hits.total: 8 }
+    - length: { hits.hits: 2 }
+    - match: { hits.hits.0.fields.country: ["UK"] }
+    - match: { hits.hits.0.inner_hits.by_location.hits.total: 4 }
+    # 2 inner hits returned instead of requested 3 as they are collapsed by city
+    - length: { hits.hits.0.inner_hits.by_location.hits.hits : 2}
+    - match: { hits.hits.0.inner_hits.by_location.hits.hits.0._id: "8" }
+    - match: { hits.hits.0.inner_hits.by_location.hits.hits.0.fields.city: ["London"] }
+    - match: { hits.hits.0.inner_hits.by_location.hits.hits.1._id: "7" }
+    - match: { hits.hits.0.inner_hits.by_location.hits.hits.1.fields.city: ["Manchester"] }
+
+    - match: { hits.hits.1.fields.country: ["Canada"] }
+    - match: { hits.hits.1.inner_hits.by_location.hits.total: 4 }
+    # 2 inner hits returned instead of requested 3 as they are collapsed by city
+    - length: { hits.hits.1.inner_hits.by_location.hits.hits : 2 }
+    - match: { hits.hits.1.inner_hits.by_location.hits.hits.0._id: "1" }
+    - match: { hits.hits.1.inner_hits.by_location.hits.hits.0.fields.city: ["Saskatoon"] }
+    - match: { hits.hits.1.inner_hits.by_location.hits.hits.1._id: "3" }
+    - match: { hits.hits.1.inner_hits.by_location.hits.hits.1.fields.city: ["Toronto"] }
+
+
+    # ************* sorted
+    - do:
+        search:
+          index: addresses
+          body:
+            query: { "match" : { "address" : "victoria" }}
+            collapse:
+              field: country
+              inner_hits:
+                name: by_location
+                size: 3
+                sort: [{ "city": "desc" }]
+                collapse:
+                  field : city
+
+    - match: { hits.total: 8 }
+    - length: { hits.hits: 2 }
+    - match: { hits.hits.0.fields.country: ["UK"] }
+    - match: { hits.hits.0.inner_hits.by_location.hits.total: 4 }
+    # 2 inner hits returned instead of requested 3 as they are collapsed by city
+    - length: { hits.hits.0.inner_hits.by_location.hits.hits : 2}
+    - match: { hits.hits.0.inner_hits.by_location.hits.hits.0._id: "7" }
+    - match: { hits.hits.0.inner_hits.by_location.hits.hits.0.fields.city: ["Manchester"] }
+    - match: { hits.hits.0.inner_hits.by_location.hits.hits.1._id: "5" }
+    - match: { hits.hits.0.inner_hits.by_location.hits.hits.1.fields.city: ["London"] }
+
+    - match: { hits.hits.1.fields.country: ["Canada"] }
+    - match: { hits.hits.1.inner_hits.by_location.hits.total: 4 }
+    # 2 inner hits returned instead of requested 3 as they are collapsed by city
+    - length: { hits.hits.1.inner_hits.by_location.hits.hits : 2 }
+    - match: { hits.hits.1.inner_hits.by_location.hits.hits.0._id: "2" }
+    - match: { hits.hits.1.inner_hits.by_location.hits.hits.0.fields.city: ["Toronto"] }
+    - match: { hits.hits.1.inner_hits.by_location.hits.hits.1._id: "1" }
+    - match: { hits.hits.1.inner_hits.by_location.hits.hits.1.fields.city: ["Saskatoon"] }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
index a6a99137dc9..917ff06c573 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
@@ -87,7 +87,8 @@ final class ExpandSearchPhase extends SearchPhase {
                     groupQuery.must(origQuery);
                 }
                 for (InnerHitBuilder innerHitBuilder : innerHitBuilders) {
-                    SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder)
+                    CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder();
+                    SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder)
                         .query(groupQuery)
                         .postFilter(searchRequest.source().postFilter());
                     SearchRequest groupRequest = buildExpandSearchRequest(searchRequest, sourceBuilder);
@@ -135,7 +136,7 @@ final class ExpandSearchPhase extends SearchPhase {
         return groupRequest;
     }
 
-    private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder options) {
+    private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder options, CollapseBuilder innerCollapseBuilder) {
         SearchSourceBuilder groupSource = new SearchSourceBuilder();
         groupSource.from(options.getFrom());
         groupSource.size(options.getSize());
@@ -167,6 +168,9 @@ final class ExpandSearchPhase extends SearchPhase {
         groupSource.explain(options.isExplain());
         groupSource.trackScores(options.isTrackScores());
         groupSource.version(options.isVersion());
+        if (innerCollapseBuilder != null) {
+            groupSource.collapse(innerCollapseBuilder);
+        }
         return groupSource;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 92da1bc3b65..6bdc55d31cd 100644
--- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -37,6 +37,7 @@ import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.FieldAndFor
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -55,6 +56,8 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
     public static final ParseField NAME_FIELD = new ParseField("name");
     public static final ParseField IGNORE_UNMAPPED = new ParseField("ignore_unmapped");
     public static final QueryBuilder DEFAULT_INNER_HIT_QUERY = new MatchAllQueryBuilder();
+    public static final ParseField COLLAPSE_FIELD = new ParseField("collapse");
+    public static final ParseField FIELD_FIELD = new ParseField("field");
 
     private static final ObjectParser<InnerHitBuilder, Void> PARSER = new ObjectParser<>("inner_hits", InnerHitBuilder::new);
 
@@ -91,6 +94,28 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
         }, SearchSourceBuilder._SOURCE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING);
         PARSER.declareObject(InnerHitBuilder::setHighlightBuilder, (p, c) -> HighlightBuilder.fromXContent(p),
                 SearchSourceBuilder.HIGHLIGHT_FIELD);
+        PARSER.declareField((parser, builder, context) -> {
+            Boolean isParsedCorrectly = false;
+            String field;
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                if (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                    if (FIELD_FIELD.match(parser.currentName(), parser.getDeprecationHandler())) {
+                        if (parser.nextToken() == XContentParser.Token.VALUE_STRING){
+                            field = parser.text();
+                            if (parser.nextToken() == XContentParser.Token.END_OBJECT){
+                                isParsedCorrectly = true;
+                                CollapseBuilder cb = new CollapseBuilder(field);
+                                builder.setInnerCollapse(cb);
+                            }
+                        }
+                    }
+                }
+            }
+            if (isParsedCorrectly == false) {
+                throw new ParsingException(parser.getTokenLocation(), "Invalid token in the inner collapse");
+            }
+
+        }, COLLAPSE_FIELD, ObjectParser.ValueType.OBJECT);
     }
 
     private String name;
@@ -109,6 +134,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
     private Set<ScriptField> scriptFields;
     private HighlightBuilder highlightBuilder;
     private FetchSourceContext fetchSourceContext;
+    private CollapseBuilder innerCollapseBuilder = null;
 
     public InnerHitBuilder() {
         this.name = null;
@@ -173,6 +199,9 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
             boolean hasChildren = in.readBoolean();
             assert hasChildren == false;
         }
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            this.innerCollapseBuilder = in.readOptionalWriteable(CollapseBuilder::new);
+        }
     }
 
     @Override
@@ -218,6 +247,9 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
             }
         }
         out.writeOptionalWriteable(highlightBuilder);
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeOptionalWriteable(innerCollapseBuilder);
+        }
     }
 
     /**
@@ -501,6 +533,15 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
         return query;
     }
 
+    public InnerHitBuilder setInnerCollapse(CollapseBuilder innerCollapseBuilder) {
+        this.innerCollapseBuilder = innerCollapseBuilder;
+        return this;
+    }
+
+    public CollapseBuilder getInnerCollapseBuilder() {
+        return innerCollapseBuilder;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -550,6 +591,9 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
         if (highlightBuilder != null) {
             builder.field(SearchSourceBuilder.HIGHLIGHT_FIELD.getPreferredName(), highlightBuilder, params);
         }
+        if (innerCollapseBuilder != null) {
+            builder.field(COLLAPSE_FIELD.getPreferredName(), innerCollapseBuilder);
+        }
         builder.endObject();
         return builder;
     }
@@ -572,13 +616,14 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
                 Objects.equals(scriptFields, that.scriptFields) &&
                 Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
                 Objects.equals(sorts, that.sorts) &&
-                Objects.equals(highlightBuilder, that.highlightBuilder);
+                Objects.equals(highlightBuilder, that.highlightBuilder) &&
+                Objects.equals(innerCollapseBuilder, that.innerCollapseBuilder);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(name, ignoreUnmapped, from, size, explain, version, trackScores,
-                storedFieldsContext, docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder);
+                storedFieldsContext, docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, innerCollapseBuilder);
     }
 
     public static InnerHitBuilder fromXContent(XContentParser parser) throws IOException {

From 82cdb574cf746ef2c4336181a8399b8bcb11ff3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Fri, 13 Jul 2018 17:41:28 +0200
Subject: [PATCH 009/260] Mute ML AutodetectMemoryLimitIT#testTooManyPartitions
 on Windows (#32044)

Adding assumption to not run this test on Windows temporarily.

Relates to #32033
---
 .../xpack/ml/integration/AutodetectMemoryLimitIT.java           | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
index 2a332fb05d5..03860ea9ae0 100644
--- a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
+++ b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.integration;
 
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
@@ -38,6 +39,7 @@ public class AutodetectMemoryLimitIT extends MlNativeAutodetectIntegTestCase {
     }
 
     public void testTooManyPartitions() throws Exception {
+        assumeFalse("AwaitsFix(bugUrl = \"https://github.com/elastic/elasticsearch/issues/32033\")", Constants.WINDOWS);
         Detector.Builder detector = new Detector.Builder("count", null);
         detector.setPartitionFieldName("user");
 

From c1a81e552fa014c5654c42bcda002eae7d10236c Mon Sep 17 00:00:00 2001
From: Michael Basnight <mbasnight@gmail.com>
Date: Fri, 13 Jul 2018 11:12:03 -0500
Subject: [PATCH 010/260] Watcher: cleanup ensureWatchExists use (#31926)

Previously, the ensureWatchExists was overridable. This commit makes
it final so that it cannot be overridden, and cleans up some redundant
code in the process.
---
 .../execution/WatchExecutionContext.java      |  2 +-
 .../execution/ManualExecutionContext.java     | 21 +++++--------------
 2 files changed, 6 insertions(+), 17 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionContext.java
index 4cdd4bb0e35..62216ff681e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionContext.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionContext.java
@@ -82,7 +82,7 @@ public abstract class WatchExecutionContext {
         return watch;
     }
 
-    public void ensureWatchExists(CheckedSupplier<Watch, Exception> supplier) throws Exception {
+    public final void ensureWatchExists(CheckedSupplier<Watch, Exception> supplier) throws Exception {
         if (watch == null) {
             watch = supplier.get();
         }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ManualExecutionContext.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ManualExecutionContext.java
index c161b24e856..abf1e5aec0d 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ManualExecutionContext.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ManualExecutionContext.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.execution;
 
-import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.watcher.actions.Action;
 import org.elasticsearch.xpack.core.watcher.actions.ActionWrapper;
@@ -29,18 +28,19 @@ public class ManualExecutionContext extends WatchExecutionContext {
     private final Map<String, ActionExecutionMode> actionModes;
     private final boolean recordExecution;
     private final boolean knownWatch;
-    private final Watch watch;
 
     ManualExecutionContext(Watch watch, boolean knownWatch, DateTime executionTime, ManualTriggerEvent triggerEvent,
                            TimeValue defaultThrottlePeriod, Input.Result inputResult, Condition.Result conditionResult,
-                           Map<String, ActionExecutionMode> actionModes, boolean recordExecution) {
+                           Map<String, ActionExecutionMode> actionModes, boolean recordExecution) throws Exception {
 
         super(watch.id(), executionTime, triggerEvent, defaultThrottlePeriod);
 
         this.actionModes = actionModes;
         this.recordExecution = recordExecution;
         this.knownWatch = knownWatch;
-        this.watch = watch;
+
+        // set the watch early to ensure calls to watch() below succeed.
+        super.ensureWatchExists(() -> watch);
 
         if (inputResult != null) {
             onInputResult(inputResult);
@@ -66,12 +66,6 @@ public class ManualExecutionContext extends WatchExecutionContext {
         }
     }
 
-    // a noop operation, as the watch is already loaded via ctor
-    @Override
-    public void ensureWatchExists(CheckedSupplier<Watch, Exception> supplier) throws Exception {
-        super.ensureWatchExists(() -> watch);
-    }
-
     @Override
     public boolean knownWatch() {
         return knownWatch;
@@ -107,11 +101,6 @@ public class ManualExecutionContext extends WatchExecutionContext {
         return recordExecution;
     }
 
-    @Override
-    public Watch watch() {
-        return watch;
-    }
-
     public static Builder builder(Watch watch, boolean knownWatch, ManualTriggerEvent event, TimeValue defaultThrottlePeriod) {
         return new Builder(watch, knownWatch, event, defaultThrottlePeriod);
     }
@@ -173,7 +162,7 @@ public class ManualExecutionContext extends WatchExecutionContext {
             return this;
         }
 
-        public ManualExecutionContext build() {
+        public ManualExecutionContext build() throws Exception {
             if (executionTime == null) {
                 executionTime = DateTime.now(DateTimeZone.UTC);
             }

From bf7689071b4e580cc3056ac231c0329fabd7f01d Mon Sep 17 00:00:00 2001
From: Michael Basnight <mbasnight@gmail.com>
Date: Fri, 13 Jul 2018 11:13:10 -0500
Subject: [PATCH 011/260] Add secure setting for watcher email password
 (#31620)

Other watcher actions already account for secure settings in their
sensitive settings, whereas the email sending action did not. This adds
the ability to optionally set a secure_password for email accounts.
---
 .../watcher/notification/email/Account.java   | 40 ++++++++++++++++---
 .../notification/email/EmailService.java      | 10 ++++-
 .../notification/email/AccountTests.java      | 30 +++++++++++++-
 3 files changed, 72 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Account.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Account.java
index 8ba8d030524..02c0e1167dd 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Account.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Account.java
@@ -7,6 +7,9 @@ package org.elasticsearch.xpack.watcher.notification.email;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.TimeValue;
@@ -24,10 +27,13 @@ import java.security.PrivilegedAction;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Properties;
+import java.util.Set;
 
 public class Account {
 
     static final String SMTP_PROTOCOL = "smtp";
+    private static final String SMTP_PASSWORD = "password";
+    private static final Setting<SecureString> SECURE_PASSWORD_SETTING = SecureSetting.secureString("secure_" + SMTP_PASSWORD, null);
 
     static {
         SecurityManager sm = System.getSecurityManager();
@@ -101,7 +107,7 @@ public class Account {
         if (auth != null && auth.password() != null) {
             password = new String(auth.password().text(cryptoService));
         } else if (config.smtp.password != null) {
-            password = new String(config.smtp.password);
+            password = new String(config.smtp.password.getChars());
         }
 
         if (profile == null) {
@@ -199,18 +205,40 @@ public class Account {
             final String host;
             final int port;
             final String user;
-            final char[] password;
+            final SecureString password;
             final Properties properties;
 
             Smtp(Settings settings) {
                 host = settings.get("host", settings.get("localaddress", settings.get("local_address")));
+
                 port = settings.getAsInt("port", settings.getAsInt("localport", settings.getAsInt("local_port", 25)));
                 user = settings.get("user", settings.get("from", null));
-                String passStr = settings.get("password", null);
-                password = passStr != null ? passStr.toCharArray() : null;
+                password = getSecureSetting(SMTP_PASSWORD, settings, SECURE_PASSWORD_SETTING);
+                //password = passStr != null ? passStr.toCharArray() : null;
                 properties = loadSmtpProperties(settings);
             }
 
+            /**
+             * Finds a setting, and then a secure setting if the setting is null, or returns null if one does not exist. This differs
+             * from other getSetting calls in that it allows for null whereas the other methods throw an exception.
+             *
+             * Note: if your setting was not previously secure, than the string reference that is in the setting object is still
+             * insecure. This is only constructing a new SecureString with the char[] of the insecure setting.
+             */
+            private static SecureString getSecureSetting(String settingName, Settings settings, Setting<SecureString> secureSetting) {
+                String value = settings.get(settingName);
+                if (value == null) {
+                    SecureString secureString = secureSetting.get(settings);
+                    if (secureString != null && secureString.length() > 0) {
+                        return secureString;
+                    } else  {
+                        return null;
+                    }
+                } else {
+                    return new SecureString(value.toCharArray());
+                }
+            }
+
             /**
              * loads the standard Java Mail properties as settings from the given account settings.
              * The standard settings are not that readable, therefore we enabled the user to configure
@@ -231,7 +259,9 @@ public class Account {
 
                 settings = builder.build();
                 Properties props = new Properties();
-                for (String key : settings.keySet()) {
+                // Secure strings can not be retreived out of a settings object and should be handled differently
+                Set<String> insecureSettings = settings.filter(s -> s.startsWith("secure_") == false).keySet();
+                for (String key : insecureSettings) {
                     props.setProperty(SMTP_SETTINGS_PREFIX + key, settings.get(key));
                 }
                 return props;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
index 3d2ea583edd..15859a5e044 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
@@ -7,6 +7,8 @@ package org.elasticsearch.xpack.watcher.notification.email;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -63,6 +65,10 @@ public class EmailService extends NotificationService<Account> {
             Setting.affixKeySetting("xpack.notification.email.account.", "smtp.password",
                     (key) -> Setting.simpleString(key, Property.Dynamic, Property.NodeScope, Property.Filtered));
 
+    private static final Setting.AffixSetting<SecureString> SETTING_SECURE_PASSWORD =
+        Setting.affixKeySetting("xpack.notification.email.account.", "smtp.secure_password",
+            (key) -> SecureSetting.secureString(key, null));
+
     private static final Setting.AffixSetting<TimeValue> SETTING_SMTP_TIMEOUT =
             Setting.affixKeySetting("xpack.notification.email.account.", "smtp.timeout",
                     (key) -> Setting.timeSetting(key, TimeValue.timeValueMinutes(2), Property.Dynamic, Property.NodeScope));
@@ -111,6 +117,7 @@ public class EmailService extends NotificationService<Account> {
         clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_PORT, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_USER, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_PASSWORD, (s, o) -> {}, (s, o) -> {});
+        clusterSettings.addAffixUpdateConsumer(SETTING_SECURE_PASSWORD, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_TIMEOUT, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_CONNECTION_TIMEOUT, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_WRITE_TIMEOUT, (s, o) -> {}, (s, o) -> {});
@@ -172,7 +179,8 @@ public class EmailService extends NotificationService<Account> {
         return Arrays.asList(SETTING_DEFAULT_ACCOUNT, SETTING_PROFILE, SETTING_EMAIL_DEFAULTS, SETTING_SMTP_AUTH, SETTING_SMTP_HOST,
                 SETTING_SMTP_PASSWORD, SETTING_SMTP_PORT, SETTING_SMTP_STARTTLS_ENABLE, SETTING_SMTP_USER, SETTING_SMTP_STARTTLS_REQUIRED,
                 SETTING_SMTP_TIMEOUT, SETTING_SMTP_CONNECTION_TIMEOUT, SETTING_SMTP_WRITE_TIMEOUT, SETTING_SMTP_LOCAL_ADDRESS,
-                SETTING_SMTP_LOCAL_PORT, SETTING_SMTP_SEND_PARTIAL, SETTING_SMTP_WAIT_ON_QUIT, SETTING_SMTP_SSL_TRUST_ADDRESS);
+                SETTING_SMTP_LOCAL_PORT, SETTING_SMTP_SEND_PARTIAL, SETTING_SMTP_WAIT_ON_QUIT, SETTING_SMTP_SSL_TRUST_ADDRESS,
+                SETTING_SECURE_PASSWORD);
     }
 
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/AccountTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/AccountTests.java
index 8e83d30ffa5..1cbaecef8fe 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/AccountTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/AccountTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.notification.email;
 
+import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
@@ -16,7 +17,6 @@ import org.junit.Before;
 import javax.mail.Address;
 import javax.mail.Message;
 import javax.mail.internet.InternetAddress;
-
 import java.util.Properties;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -149,7 +149,7 @@ public class AccountTests extends ESTestCase {
         assertThat(config.smtp.host, is(host));
         assertThat(config.smtp.user, is(user));
         if (password != null) {
-            assertThat(config.smtp.password, is(password.toCharArray()));
+            assertThat(config.smtp.password.getChars(), is(password.toCharArray()));
         } else {
             assertThat(config.smtp.password, nullValue());
         }
@@ -292,4 +292,30 @@ public class AccountTests extends ESTestCase {
                     .build()), null, logger);
         });
     }
+
+    public void testEnsurePasswordSetAsSecureSetting() {
+        String password = "password";
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("smtp.secure_password", password);
+
+        Settings settings = Settings.builder()
+            .put("smtp.host", "localhost")
+            .put("smtp.port", server.port())
+            .put("smtp.connection_timeout", TimeValue.timeValueMinutes(4))
+            .setSecureSettings(secureSettings)
+            .build();
+
+        Account.Config config = new Account.Config("default", settings);
+        assertThat(config.smtp.password.getChars(), equalTo(password.toCharArray()));
+
+        settings = Settings.builder()
+            .put("smtp.host", "localhost")
+            .put("smtp.port", server.port())
+            .put("smtp.connection_timeout", TimeValue.timeValueMinutes(4))
+            .put("smtp.password", password)
+            .build();
+
+        config = new Account.Config("default", settings);
+        assertThat(config.smtp.password.getChars(), equalTo(password.toCharArray()));
+    }
 }

From 2c3ea43f45a6364b215acca428f29a7af7f66d10 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 13 Jul 2018 09:33:27 -0700
Subject: [PATCH 012/260] HLRC: Add xpack usage api (#31975)

This commit adds the _xpack/usage api to the high level rest client.
Currently in the transport api, the usage data is exposed in a limited
fashion, at most giving one level of helper methods for the inner keys
of data, but then exposing thos subobjects as maps of objects. Rather
than making parsers for every set of usage data from each feature, this
PR exposes the entire set of usage data as a map of maps.
---
 .../client/RequestConverters.java             |  8 +++
 .../org/elasticsearch/client/XPackClient.java | 23 ++++++++
 .../MiscellaneousDocumentationIT.java         | 49 ++++++++++++++++
 .../miscellaneous/x-pack-usage.asciidoc       | 54 ++++++++++++++++++
 .../action/TransportXPackUsageAction.java     |  1 +
 .../xpack/core/action/XPackUsageRequest.java  | 18 ------
 .../core/action/XPackUsageRequestBuilder.java |  1 +
 .../cluster/ClusterStatsCollectorTests.java   |  2 +-
 .../watcher/WatcherXpackUsageStatsTests.java  |  2 +-
 .../protocol/xpack/XPackUsageRequest.java     | 31 ++++++++++
 .../protocol/xpack/XPackUsageResponse.java    | 57 +++++++++++++++++++
 11 files changed, 226 insertions(+), 20 deletions(-)
 create mode 100644 docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequest.java
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageRequest.java
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageResponse.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index 126a9c7d4b4..9dbd4916c77 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -106,6 +106,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -1096,6 +1097,13 @@ final class RequestConverters {
         return request;
     }
 
+    static Request xpackUsage(XPackUsageRequest usageRequest) {
+        Request request = new Request(HttpGet.METHOD_NAME, "/_xpack/usage");
+        Params parameters = new Params(request);
+        parameters.withMasterTimeout(usageRequest.masterNodeTimeout());
+        return request;
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
index 5942bfa35a4..a497619b987 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
@@ -22,6 +22,8 @@ package org.elasticsearch.client;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageResponse;
 
 import java.io.IOException;
 
@@ -70,4 +72,25 @@ public final class XPackClient {
         restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xPackInfo, options,
             XPackInfoResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Fetch usage information about X-Pack features from the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public XPackUsageResponse usage(XPackUsageRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::xpackUsage, options,
+            XPackUsageResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously fetch usage information about X-Pack features from the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void usageAsync(XPackUsageRequest request, RequestOptions options, ActionListener<XPackUsageResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xpackUsage, options,
+            XPackUsageResponse::fromXContent, listener, emptySet());
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java
index 75c14097c45..a99b991620a 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java
@@ -35,12 +35,17 @@ import org.elasticsearch.protocol.xpack.XPackInfoResponse;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.BuildInfo;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageResponse;
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import static org.hamcrest.Matchers.is;
+
 /**
  * Documentation for miscellaneous APIs in the high level java client.
  * Code wrapped in {@code tag} and {@code end} tags is included in the docs.
@@ -129,6 +134,50 @@ public class MiscellaneousDocumentationIT extends ESRestHighLevelClientTestCase
         }
     }
 
+    public void testXPackUsage() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            //tag::x-pack-usage-execute
+            XPackUsageRequest request = new XPackUsageRequest();
+            XPackUsageResponse response = client.xpack().usage(request, RequestOptions.DEFAULT);
+            //end::x-pack-usage-execute
+
+            //tag::x-pack-usage-response
+            Map<String, Map<String, Object>> usages = response.getUsages();
+            Map<String, Object> monitoringUsage = usages.get("monitoring");
+            assertThat(monitoringUsage.get("available"), is(true));
+            assertThat(monitoringUsage.get("enabled"), is(true));
+            assertThat(monitoringUsage.get("collection_enabled"), is(false));
+            //end::x-pack-usage-response
+        }
+        {
+            XPackUsageRequest request = new XPackUsageRequest();
+            // tag::x-pack-usage-execute-listener
+            ActionListener<XPackUsageResponse> listener = new ActionListener<XPackUsageResponse>() {
+                @Override
+                public void onResponse(XPackUsageResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::x-pack-usage-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::x-pack-usage-execute-async
+            client.xpack().usageAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::x-pack-usage-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
     public void testInitializationFromClientBuilder() throws IOException {
         //tag::rest-high-level-client-init
         RestHighLevelClient client = new RestHighLevelClient(
diff --git a/docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc b/docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc
new file mode 100644
index 00000000000..0927ae71c0b
--- /dev/null
+++ b/docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc
@@ -0,0 +1,54 @@
+[[java-rest-high-x-pack-usage]]
+=== X-Pack Usage API
+
+[[java-rest-high-x-pack-usage-execution]]
+==== Execution
+
+Detailed information about the usage of features from {xpack} can be
+retrieved using the `usage()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-execute]
+--------------------------------------------------
+
+[[java-rest-high-x-pack-info-response]]
+==== Response
+
+The returned `XPackUsageResponse` contains a `Map` keyed by feature name.
+Every feature map has an `available` key, indicating whether that
+feature is available given the current license, and an `enabled` key,
+indicating whether that feature is currently enabled. Other keys
+are specific to each feature.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-response]
+--------------------------------------------------
+
+[[java-rest-high-x-pack-usage-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-execute-async]
+--------------------------------------------------
+<1> The call to execute the usage api and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `XPackUsageResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-usage-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
index f3abad5e68b..6b7d5b96d20 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
@@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequest.java
deleted file mode 100644
index d578249c147..00000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequest.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.action;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.support.master.MasterNodeRequest;
-
-public class XPackUsageRequest extends MasterNodeRequest<XPackUsageRequest> {
-
-    @Override
-    public ActionRequestValidationException validate() {
-        return null;
-    }
-
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequestBuilder.java
index 789460f1339..92c2ba75ec1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequestBuilder.java
@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.action;
 
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 
 public class XPackUsageRequestBuilder
         extends MasterNodeOperationRequestBuilder<XPackUsageRequest, XPackUsageResponse, XPackUsageRequestBuilder> {
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java
index 6784b00361b..49355d51495 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java
@@ -26,7 +26,7 @@ import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
-import org.elasticsearch.xpack.core.action.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherXpackUsageStatsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherXpackUsageStatsTests.java
index 3a314640d74..7c07c98eb47 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherXpackUsageStatsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherXpackUsageStatsTests.java
@@ -7,7 +7,7 @@ package org.elasticsearch.xpack.watcher;
 
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
-import org.elasticsearch.xpack.core.action.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageRequest.java
new file mode 100644
index 00000000000..f5f6d9d949b
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageRequest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+
+public class XPackUsageRequest extends MasterNodeRequest<XPackUsageRequest> {
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageResponse.java
new file mode 100644
index 00000000000..3459403bd61
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageResponse.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.protocol.xpack;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Response object from calling the xpack usage api.
+ *
+ * Usage information for each feature is accessible through {@link #getUsages()}.
+ */
+public class XPackUsageResponse {
+
+    private final Map<String, Map<String, Object>> usages;
+
+    private XPackUsageResponse(Map<String, Map<String, Object>> usages) throws IOException {
+        this.usages = usages;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> castMap(Object value) {
+        return (Map<String, Object>)value;
+    }
+
+    /** Return a map from feature name to usage information for that feature. */
+    public Map<String, Map<String, Object>> getUsages() {
+        return usages;
+    }
+
+    public static XPackUsageResponse fromXContent(XContentParser parser) throws IOException {
+        Map<String, Object> rawMap = parser.map();
+        Map<String, Map<String, Object>> usages = rawMap.entrySet().stream().collect(
+            Collectors.toMap(Map.Entry::getKey, e -> castMap(e.getValue())));
+        return new XPackUsageResponse(usages);
+    }
+}

From 0edb096eb4763e2149c6c67aa8842198ecefb108 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Fri, 13 Jul 2018 18:08:35 +0100
Subject: [PATCH 013/260] Adds a new auto-interval date histogram (#28993)

* Adds a new auto-interval date histogram

This change adds a new type of histogram aggregation called `auto_date_histogram` where you can specify the target number of buckets you require and it will find an appropriate interval for the returned buckets. The aggregation works by first collecting documents in buckets at second interval, when it has created more than the target number of buckets it merges these buckets into minute interval bucket and continues collecting until it reaches the target number of buckets again. It will keep merging buckets when it exceeds the target until either collection is finished or the highest interval (currently years) is reached. A similar process happens at reduce time.

This aggregation intentionally does not support min_doc_count, offest and extended_bounds to keep the already complex logic from becoming more complex. The aggregation accepts sub-aggregations but will always operate in `breadth_first` mode deferring the computation of sub-aggregations until the final buckets from the shard are known. min_doc_count is effectively hard-coded to zero meaning that we will insert empty buckets where necessary.

Closes #9572

* Adds documentation

* Added sub aggregator test

* Fixes failing docs test

* Brings branch up to date with master changes

* trying to get tests to pass again

* Fixes multiBucketConsumer accounting

* Collects more buckets than needed on shards

This gives us more options at reduce time in terms of how we do the
final merge of the buckeets to produce the final result

* Revert "Collects more buckets than needed on shards"

This reverts commit 993c782d117892af9a3c86a51921cdee630a3ac5.

* Adds ability to merge within a rounding

* Fixes nonn-timezone doc test failure

* Fix time zone tests

* iterates on tests

* Adds test case and documentation changes

Added some notes in the documentation about the intervals that can bbe
returned.

Also added a test case that utilises the merging of conseecutive buckets

* Fixes performance bug

The bug meant that getAppropriate rounding look a huge amount of time
if the range of the data was large but also sparsely populated. In
these situations the rounding would be very low so iterating through
the rounding values from the min key to the max keey look a long time
(~120 seconds in one test).

The solution is to add a rough estimate first which chooses the
rounding based just on the long values of the min and max keeys alone
but selects the rounding one lower than the one it thinks is
appropriate so the accurate method can choose the final rounding taking
into account the fact that intervals are not always fixed length.

Thee commit also adds more tests

* Changes to only do complex reduction on final reduce

* merge latest with master

* correct tests and add a new test case for 10k buckets

* refactor to perform bucket number check in innerBuild

* correctly derive bucket setting, update tests to increase bucket threshold

* fix checkstyle

* address code review comments

* add documentation for default buckets

* fix typo
---
 .../client/RestHighLevelClient.java           |    3 +
 docs/reference/aggregations/bucket.asciidoc   |    2 +
 .../autodatehistogram-aggregation.asciidoc    |  283 ++++
 .../elasticsearch/search/SearchModule.java    |    4 +
 .../bucket/BucketsAggregator.java             |   13 +
 .../MergingBucketsDeferringCollector.java     |  236 +++
 .../AutoDateHistogramAggregationBuilder.java  |  218 +++
 .../AutoDateHistogramAggregator.java          |  199 +++
 .../AutoDateHistogramAggregatorFactory.java   |   72 +
 .../histogram/DateHistogramAggregator.java    |    4 +-
 .../histogram/InternalAutoDateHistogram.java  |  601 ++++++++
 .../histogram/InternalDateHistogram.java      |    2 +-
 .../histogram/ParsedAutoDateHistogram.java    |   91 ++
 .../aggregations/AggregationsTests.java       |    2 +
 .../bucket/AutoDateHistogramTests.java        |   44 +
 .../AutoDateHistogramAggregatorTests.java     | 1332 +++++++++++++++++
 .../InternalAutoDateHistogramTests.java       |  154 ++
 .../aggregations/AggregatorTestCase.java      |    3 +-
 .../test/InternalAggregationTestCase.java     |    3 +
 ...nternalMultiBucketAggregationTestCase.java |    3 +-
 20 files changed, 3263 insertions(+), 6 deletions(-)
 create mode 100644 docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedAutoDateHistogram.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/AutoDateHistogramTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index df674ea898e..b9e41b87932 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -85,8 +85,10 @@ import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBu
 import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedAutoDateHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram;
 import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
@@ -1004,6 +1006,7 @@ public class RestHighLevelClient implements Closeable {
         map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
         map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
         map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));
+        map.put(AutoDateHistogramAggregationBuilder.NAME, (p, c) -> ParsedAutoDateHistogram.fromXContent(p, (String) c));
         map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
         map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
         map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc
index 1233e0d9b73..ddb55e8d34c 100644
--- a/docs/reference/aggregations/bucket.asciidoc
+++ b/docs/reference/aggregations/bucket.asciidoc
@@ -19,6 +19,8 @@ the limit will fail with an exception.
 
 include::bucket/adjacency-matrix-aggregation.asciidoc[]
 
+include::bucket/autodatehistogram-aggregation.asciidoc[]
+
 include::bucket/children-aggregation.asciidoc[]
 
 include::bucket/composite-aggregation.asciidoc[]
diff --git a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
new file mode 100644
index 00000000000..28cb65ce6cc
--- /dev/null
+++ b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
@@ -0,0 +1,283 @@
+[[search-aggregations-bucket-autodatehistogram-aggregation]]
+=== Auto-interval Date Histogram Aggregation
+
+A multi-bucket aggregation similar to the <<search-aggregations-bucket-datehistogram-aggregation>> except 
+instead of providing an interval to use as the width of each bucket, a target number of buckets is provided
+indicating the number of buckets needed and the interval of the buckets is automatically chosen to best achieve
+that target. The number of buckets returned will always be less than or equal to this target number.
+
+The buckets field is optional, and will default to 10 buckets if not specified.
+
+Requesting a target of 10 buckets.
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+    "aggs" : {
+        "sales_over_time" : {
+            "auto_date_histogram" : {
+                "field" : "date",
+                "buckets" : 10
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
+==== Keys
+
+Internally, a date is represented as a 64 bit number representing a timestamp
+in milliseconds-since-the-epoch. These timestamps are returned as the bucket
+++key++s. The `key_as_string` is the same timestamp converted to a formatted
+date string using the format specified with the `format` parameter:
+
+TIP: If no `format` is specified, then it will use the first date
+<<mapping-date-format,format>> specified in the field mapping.
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+    "aggs" : {
+        "sales_over_time" : {
+            "auto_date_histogram" : {
+                "field" : "date",
+                "buckets" : 5,
+                "format" : "yyyy-MM-dd" <1>
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
+<1> Supports expressive date <<date-format-pattern,format pattern>>
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "sales_over_time": {
+            "buckets": [
+                {
+                    "key_as_string": "2015-01-01",
+                    "key": 1420070400000,
+                    "doc_count": 3
+                },
+                {
+                    "key_as_string": "2015-02-01",
+                    "key": 1422748800000,
+                    "doc_count": 2
+                },
+                {
+                    "key_as_string": "2015-03-01",
+                    "key": 1425168000000,
+                    "doc_count": 2
+                }
+            ]
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+=== Intervals
+
+The interval of the returned buckets is selected based on the data collected by the 
+aggregation so that the number of buckets returned is less than or equal to the number 
+requested. The possible intervals returned are:
+
+[horizontal]
+seconds::      In multiples of 1, 5, 10 and 30
+minutes::      In multiples of 1, 5, 10 and 30
+hours::        In multiples of 1, 3 and 12
+days::         In multiples of 1, and 7
+months::       In multiples of 1, and 3
+years::        In multiples of 1, 5, 10, 20, 50 and 100
+
+In the worst case, where the number of daily buckets are too many for the requested 
+number of buckets, the number of buckets returned will be 1/7th of the number of 
+buckets requested.
+
+==== Time Zone
+
+Date-times are stored in Elasticsearch in UTC.  By default, all bucketing and
+rounding is also done in UTC. The `time_zone` parameter can be used to indicate
+that bucketing should use a different time zone.
+
+Time zones may either be specified as an ISO 8601 UTC offset (e.g. `+01:00` or
+`-08:00`)  or as a timezone id, an identifier used in the TZ database like
+`America/Los_Angeles`.
+
+Consider the following example:
+
+[source,js]
+---------------------------------
+PUT my_index/log/1?refresh
+{
+  "date": "2015-10-01T00:30:00Z"
+}
+
+PUT my_index/log/2?refresh
+{
+  "date": "2015-10-01T01:30:00Z"
+}
+
+PUT my_index/log/3?refresh
+{
+  "date": "2015-10-01T02:30:00Z"
+}
+
+GET my_index/_search?size=0
+{
+  "aggs": {
+    "by_day": {
+      "auto_date_histogram": {
+        "field":     "date",
+        "buckets" : 3
+      }
+    }
+  }
+}
+---------------------------------
+// CONSOLE
+
+UTC is used if no time zone is specified, three 1-hour buckets are returned 
+starting at midnight UTC on 1 October 2015:
+
+[source,js]
+---------------------------------
+{
+  ...
+  "aggregations": {
+    "by_day": {
+      "buckets": [
+        {
+          "key_as_string": "2015-10-01T00:00:00.000Z",
+          "key": 1443657600000,
+          "doc_count": 1
+        },
+        {
+          "key_as_string": "2015-10-01T01:00:00.000Z",
+          "key": 1443661200000,
+          "doc_count": 1
+        },
+        {
+          "key_as_string": "2015-10-01T02:00:00.000Z",
+          "key": 1443664800000,
+          "doc_count": 1
+        }
+      ]
+    }
+  }
+}
+---------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+If a `time_zone` of `-01:00` is specified, then midnight starts at one hour before
+midnight UTC:
+
+[source,js]
+---------------------------------
+GET my_index/_search?size=0
+{
+  "aggs": {
+    "by_day": {
+      "auto_date_histogram": {
+        "field":     "date",
+        "buckets" : 3,
+        "time_zone": "-01:00"
+      }
+    }
+  }
+}
+---------------------------------
+// CONSOLE
+// TEST[continued]
+
+
+Now three 1-hour buckets are still returned but the first bucket starts at 
+11:00pm on 30 September 2015 since that is the local time for the bucket in 
+the specified time zone.
+
+[source,js]
+---------------------------------
+{
+  ...
+  "aggregations": {
+    "by_day": {
+      "buckets": [
+        {
+          "key_as_string": "2015-09-30T23:00:00.000-01:00",
+          "key": 1443657600000,
+          "doc_count": 1
+        },
+        {
+          "key_as_string": "2015-10-01T00:00:00.000-01:00",
+          "key": 1443661200000,
+          "doc_count": 1
+        },
+        {
+          "key_as_string": "2015-10-01T01:00:00.000-01:00",
+          "key": 1443664800000,
+          "doc_count": 1
+        }
+      ]
+    }
+  }
+}
+---------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+<1> The `key_as_string` value represents midnight on each day
+    in the specified time zone.
+
+WARNING: When using time zones that follow DST (daylight savings time) changes,
+buckets close to the moment when those changes happen can have slightly different
+sizes than neighbouring buckets.
+For example, consider a DST start in the `CET` time zone: on 27 March 2016 at 2am,
+clocks were turned forward 1 hour to 3am local time. If the result of the aggregation 
+was daily buckets, the bucket covering that day will only hold data for 23 hours 
+instead of the usual 24 hours for other buckets. The same is true for shorter intervals 
+like e.g. 12h. Here, we will have only a 11h bucket on the morning of 27 March when the 
+DST shift happens.
+
+==== Scripts
+
+Like with the normal <<search-aggregations-bucket-datehistogram-aggregation, `date_histogram`>>, both document level 
+scripts and value level scripts are supported. This aggregation does not however, support the `min_doc_count`, 
+`extended_bounds` and `order` parameters.  
+
+==== Missing value
+
+The `missing` parameter defines how documents that are missing a value should be treated.
+By default they will be ignored but it is also possible to treat them as if they
+had a value.
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+    "aggs" : {
+        "sale_date" : {
+             "auto_date_histogram" : {
+                 "field" : "date",
+                 "buckets": 10,
+                 "missing": "2000/01/01" <1>
+             }
+         }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
+<1> Documents without a value in the `publish_date` field will fall into the same bucket as documents that have the value `2000-01-01`.
+
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index 199d2278bf7..efef1aeb04f 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -109,8 +109,10 @@ import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBu
 import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalAutoDateHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
 import org.elasticsearch.search.aggregations.bucket.missing.InternalMissing;
@@ -395,6 +397,8 @@ public class SearchModule {
                 HistogramAggregationBuilder::parse).addResultReader(InternalHistogram::new));
         registerAggregation(new AggregationSpec(DateHistogramAggregationBuilder.NAME, DateHistogramAggregationBuilder::new,
                 DateHistogramAggregationBuilder::parse).addResultReader(InternalDateHistogram::new));
+        registerAggregation(new AggregationSpec(AutoDateHistogramAggregationBuilder.NAME, AutoDateHistogramAggregationBuilder::new,
+                AutoDateHistogramAggregationBuilder::parse).addResultReader(InternalAutoDateHistogram::new));
         registerAggregation(new AggregationSpec(GeoDistanceAggregationBuilder.NAME, GeoDistanceAggregationBuilder::new,
                 GeoDistanceAggregationBuilder::parse).addResultReader(InternalGeoDistance::new));
         registerAggregation(new AggregationSpec(GeoGridAggregationBuilder.NAME, GeoGridAggregationBuilder::new,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
index 504758e7a4e..7b09ac9d618 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
@@ -84,6 +84,19 @@ public abstract class BucketsAggregator extends AggregatorBase {
         subCollector.collect(doc, bucketOrd);
     }
 
+    public final void mergeBuckets(long[] mergeMap, long newNumBuckets) {
+        try (IntArray oldDocCounts = docCounts) {
+            docCounts = bigArrays.newIntArray(newNumBuckets, true);
+            docCounts.fill(0, newNumBuckets, 0);
+            for (int i = 0; i < oldDocCounts.size(); i++) {
+                int docCount = oldDocCounts.get(i);
+                if (docCount != 0) {
+                    docCounts.increment(mergeMap[i], docCount);
+                }
+            }
+        }
+    }
+
     public IntArray getDocCounts() {
         return docCounts;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
new file mode 100644
index 00000000000..f357e9d286f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedLongValues;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.LongHash;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.BucketCollector;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A specialization of {@link DeferringBucketCollector} that collects all
+ * matches and then is able to replay a given subset of buckets. Exposes
+ * mergeBuckets, which can be invoked by the aggregator when increasing the
+ * rounding interval.
+ */
+public class MergingBucketsDeferringCollector extends DeferringBucketCollector {
+
+    List<Entry> entries = new ArrayList<>();
+    BucketCollector collector;
+    final SearchContext searchContext;
+    LeafReaderContext context;
+    PackedLongValues.Builder docDeltas;
+    PackedLongValues.Builder buckets;
+    long maxBucket = -1;
+    boolean finished = false;
+    LongHash selectedBuckets;
+
+    public MergingBucketsDeferringCollector(SearchContext context) {
+        this.searchContext = context;
+    }
+
+    @Override
+    public void setDeferredCollector(Iterable<BucketCollector> deferredCollectors) {
+        this.collector = BucketCollector.wrap(deferredCollectors);
+    }
+
+    @Override
+    public boolean needsScores() {
+        if (collector == null) {
+            throw new IllegalStateException();
+        }
+        return collector.needsScores();
+    }
+
+    @Override
+    public void preCollection() throws IOException {
+        collector.preCollection();
+    }
+
+    private void finishLeaf() {
+        if (context != null) {
+            entries.add(new Entry(context, docDeltas.build(), buckets.build()));
+        }
+        context = null;
+        docDeltas = null;
+        buckets = null;
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException {
+        finishLeaf();
+
+        context = ctx;
+        docDeltas = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+        buckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+
+        return new LeafBucketCollector() {
+            int lastDoc = 0;
+
+            @Override
+            public void collect(int doc, long bucket) {
+                docDeltas.add(doc - lastDoc);
+                buckets.add(bucket);
+                lastDoc = doc;
+                maxBucket = Math.max(maxBucket, bucket);
+            }
+        };
+    }
+
+    public void mergeBuckets(long[] mergeMap) {
+
+        List<Entry> newEntries = new ArrayList<>(entries.size());
+        for (Entry sourceEntry : entries) {
+            PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+            for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) {
+                long bucket = itr.next();
+                newBuckets.add(mergeMap[Math.toIntExact(bucket)]);
+            }
+            newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build()));
+        }
+        entries = newEntries;
+
+        // if there are buckets that have been collected in the current segment
+        // we need to update the bucket ordinals there too
+        if (buckets.size() > 0) {
+            PackedLongValues currentBuckets = buckets.build();
+            PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+            for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) {
+                long bucket = itr.next();
+                newBuckets.add(mergeMap[Math.toIntExact(bucket)]);
+            }
+            buckets = newBuckets;
+        }
+    }
+
+    @Override
+    public void postCollection() {
+        finishLeaf();
+        finished = true;
+    }
+
+    /**
+     * Replay the wrapped collector, but only on a selection of buckets.
+     */
+    @Override
+    public void prepareSelectedBuckets(long... selectedBuckets) throws IOException {
+        if (finished == false) {
+            throw new IllegalStateException("Cannot replay yet, collection is not finished: postCollect() has not been called");
+        }
+        if (this.selectedBuckets != null) {
+            throw new IllegalStateException("Already been replayed");
+        }
+
+        final LongHash hash = new LongHash(selectedBuckets.length, BigArrays.NON_RECYCLING_INSTANCE);
+        for (long bucket : selectedBuckets) {
+            hash.add(bucket);
+        }
+        this.selectedBuckets = hash;
+
+        boolean needsScores = collector.needsScores();
+        Weight weight = null;
+        if (needsScores) {
+            weight = searchContext.searcher().createNormalizedWeight(searchContext.query(), true);
+        }
+        for (Entry entry : entries) {
+            final LeafBucketCollector leafCollector = collector.getLeafCollector(entry.context);
+            DocIdSetIterator docIt = null;
+            if (needsScores && entry.docDeltas.size() > 0) {
+                Scorer scorer = weight.scorer(entry.context);
+                // We don't need to check if the scorer is null
+                // since we are sure that there are documents to replay
+                // (entry.docDeltas it not empty).
+                docIt = scorer.iterator();
+                leafCollector.setScorer(scorer);
+            }
+            final PackedLongValues.Iterator docDeltaIterator = entry.docDeltas.iterator();
+            final PackedLongValues.Iterator buckets = entry.buckets.iterator();
+            int doc = 0;
+            for (long i = 0, end = entry.docDeltas.size(); i < end; ++i) {
+                doc += docDeltaIterator.next();
+                final long bucket = buckets.next();
+                final long rebasedBucket = hash.find(bucket);
+                if (rebasedBucket != -1) {
+                    if (needsScores) {
+                        if (docIt.docID() < doc) {
+                            docIt.advance(doc);
+                        }
+                        // aggregations should only be replayed on matching
+                        // documents
+                        assert docIt.docID() == doc;
+                    }
+                    leafCollector.collect(doc, rebasedBucket);
+                }
+            }
+        }
+
+        collector.postCollection();
+    }
+
+    /**
+     * Wrap the provided aggregator so that it behaves (almost) as if it had
+     * been collected directly.
+     */
+    @Override
+    public Aggregator wrap(final Aggregator in) {
+
+        return new WrappedAggregator(in) {
+
+            @Override
+            public InternalAggregation buildAggregation(long bucket) throws IOException {
+                if (selectedBuckets == null) {
+                    throw new IllegalStateException("Collection has not been replayed yet.");
+                }
+                final long rebasedBucket = selectedBuckets.find(bucket);
+                if (rebasedBucket == -1) {
+                    throw new IllegalStateException("Cannot build for a bucket which has not been collected [" + bucket + "]");
+                }
+                return in.buildAggregation(rebasedBucket);
+            }
+
+        };
+    }
+
+    private static class Entry {
+        final LeafReaderContext context;
+        final PackedLongValues docDeltas;
+        final PackedLongValues buckets;
+
+        Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongValues buckets) {
+            this.context = context;
+            this.docDeltas = docDeltas;
+            this.buckets = buckets;
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
new file mode 100644
index 00000000000..366060835d8
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.rounding.DateTimeUnit;
+import org.elasticsearch.common.rounding.Rounding;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+
+public class AutoDateHistogramAggregationBuilder
+        extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, AutoDateHistogramAggregationBuilder> {
+
+    public static final String NAME = "auto_date_histogram";
+
+    public static final ParseField NUM_BUCKETS_FIELD = new ParseField("buckets");
+
+    private static final ObjectParser<AutoDateHistogramAggregationBuilder, Void> PARSER;
+    static {
+        PARSER = new ObjectParser<>(AutoDateHistogramAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true);
+
+        PARSER.declareInt(AutoDateHistogramAggregationBuilder::setNumBuckets, NUM_BUCKETS_FIELD);
+    }
+
+    public static AutoDateHistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
+        return PARSER.parse(parser, new AutoDateHistogramAggregationBuilder(aggregationName), null);
+    }
+
+    private int numBuckets = 10;
+
+    /** Create a new builder with the given name. */
+    public AutoDateHistogramAggregationBuilder(String name) {
+        super(name, ValuesSourceType.NUMERIC, ValueType.DATE);
+    }
+
+    /** Read from a stream, for internal use only. */
+    public AutoDateHistogramAggregationBuilder(StreamInput in) throws IOException {
+        super(in, ValuesSourceType.NUMERIC, ValueType.DATE);
+        numBuckets = in.readVInt();
+    }
+
+    protected AutoDateHistogramAggregationBuilder(AutoDateHistogramAggregationBuilder clone, Builder factoriesBuilder,
+            Map<String, Object> metaData) {
+        super(clone, factoriesBuilder, metaData);
+        this.numBuckets = clone.numBuckets;
+    }
+
+    @Override
+    protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, Object> metaData) {
+        return new AutoDateHistogramAggregationBuilder(this, factoriesBuilder, metaData);
+    }
+
+    @Override
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+        out.writeVInt(numBuckets);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+
+    public AutoDateHistogramAggregationBuilder setNumBuckets(int numBuckets) {
+        if (numBuckets <= 0) {
+            throw new IllegalArgumentException(NUM_BUCKETS_FIELD.getPreferredName() + " must be greater than 0 for [" + name + "]");
+        }
+        this.numBuckets = numBuckets;
+        return this;
+    }
+
+    public int getNumBuckets() {
+        return numBuckets;
+    }
+
+    @Override
+    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
+            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+        RoundingInfo[] roundings = new RoundingInfo[6];
+        roundings[0] = new RoundingInfo(createRounding(DateTimeUnit.SECOND_OF_MINUTE), 1000L, 1, 5, 10, 30);
+        roundings[1] = new RoundingInfo(createRounding(DateTimeUnit.MINUTES_OF_HOUR), 60 * 1000L, 1, 5, 10, 30);
+        roundings[2] = new RoundingInfo(createRounding(DateTimeUnit.HOUR_OF_DAY), 60 * 60 * 1000L, 1, 3, 12);
+        roundings[3] = new RoundingInfo(createRounding(DateTimeUnit.DAY_OF_MONTH), 24 * 60 * 60 * 1000L, 1, 7);
+        roundings[4] = new RoundingInfo(createRounding(DateTimeUnit.MONTH_OF_YEAR), 30 * 24 * 60 * 60 * 1000L, 1, 3);
+        roundings[5] = new RoundingInfo(createRounding(DateTimeUnit.YEAR_OF_CENTURY), 365 * 24 * 60 * 60 * 1000L, 1, 5, 10, 20, 50, 100);
+
+        int maxRoundingInterval = Arrays.stream(roundings,0, roundings.length-1)
+            .map(rounding -> rounding.innerIntervals)
+            .flatMapToInt(Arrays::stream)
+            .boxed()
+            .reduce(Integer::max).get();
+        Settings settings = context.getQueryShardContext().getIndexSettings().getNodeSettings();
+        int maxBuckets = MultiBucketConsumerService.MAX_BUCKET_SETTING.get(settings);
+        int bucketCeiling = maxBuckets / maxRoundingInterval;
+        if (numBuckets > bucketCeiling) {
+            throw new IllegalArgumentException(NUM_BUCKETS_FIELD.getPreferredName()+
+                " must be less than " + bucketCeiling);
+        }
+        return new AutoDateHistogramAggregatorFactory(name, config, numBuckets, roundings, context, parent, subFactoriesBuilder, metaData);
+    }
+
+    private Rounding createRounding(DateTimeUnit interval) {
+        Rounding.Builder tzRoundingBuilder = Rounding.builder(interval);
+        if (timeZone() != null) {
+            tzRoundingBuilder.timeZone(timeZone());
+        }
+        Rounding rounding = tzRoundingBuilder.build();
+        return rounding;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(NUM_BUCKETS_FIELD.getPreferredName(), numBuckets);
+        return builder;
+    }
+
+    @Override
+    protected int innerHashCode() {
+        return Objects.hash(numBuckets);
+    }
+
+    @Override
+    protected boolean innerEquals(Object obj) {
+        AutoDateHistogramAggregationBuilder other = (AutoDateHistogramAggregationBuilder) obj;
+        return Objects.equals(numBuckets, other.numBuckets);
+    }
+
+    public static class RoundingInfo implements Writeable {
+        final Rounding rounding;
+        final int[] innerIntervals;
+        final long roughEstimateDurationMillis;
+
+        public RoundingInfo(Rounding rounding, long roughEstimateDurationMillis, int... innerIntervals) {
+            this.rounding = rounding;
+            this.roughEstimateDurationMillis = roughEstimateDurationMillis;
+            this.innerIntervals = innerIntervals;
+        }
+
+        public RoundingInfo(StreamInput in) throws IOException {
+            rounding = Rounding.Streams.read(in);
+            roughEstimateDurationMillis = in.readVLong();
+            innerIntervals = in.readIntArray();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            Rounding.Streams.write(rounding, out);
+            out.writeVLong(roughEstimateDurationMillis);
+            out.writeIntArray(innerIntervals);
+        }
+
+        public int getMaximumInnerInterval() {
+            return innerIntervals[innerIntervals.length - 1];
+        }
+
+        public long getRoughEstimateDurationMillis() {
+            return roughEstimateDurationMillis;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(rounding, Arrays.hashCode(innerIntervals));
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            RoundingInfo other = (RoundingInfo) obj;
+            return Objects.equals(rounding, other.rounding) &&
+                    Objects.deepEquals(innerIntervals, other.innerIntervals);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
new file mode 100644
index 00000000000..f86145386f1
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.rounding.Rounding;
+import org.elasticsearch.common.util.LongHash;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.bucket.DeferableBucketAggregator;
+import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector;
+import org.elasticsearch.search.aggregations.bucket.MergingBucketsDeferringCollector;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An aggregator for date values. Every date is rounded down using a configured
+ * {@link Rounding}.
+ *
+ * @see Rounding
+ */
+class AutoDateHistogramAggregator extends DeferableBucketAggregator {
+
+    private final ValuesSource.Numeric valuesSource;
+    private final DocValueFormat formatter;
+    private final RoundingInfo[] roundingInfos;
+    private int roundingIdx = 0;
+
+    private LongHash bucketOrds;
+    private int targetBuckets;
+    private MergingBucketsDeferringCollector deferringCollector;
+
+    AutoDateHistogramAggregator(String name, AggregatorFactories factories, int numBuckets, RoundingInfo[] roundingInfos,
+            @Nullable ValuesSource.Numeric valuesSource, DocValueFormat formatter, SearchContext aggregationContext, Aggregator parent,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
+
+        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        this.targetBuckets = numBuckets;
+        this.valuesSource = valuesSource;
+        this.formatter = formatter;
+        this.roundingInfos = roundingInfos;
+
+        bucketOrds = new LongHash(1, aggregationContext.bigArrays());
+
+    }
+
+    @Override
+    public boolean needsScores() {
+        return (valuesSource != null && valuesSource.needsScores()) || super.needsScores();
+    }
+
+    @Override
+    protected boolean shouldDefer(Aggregator aggregator) {
+        return true;
+    }
+
+    @Override
+    public DeferringBucketCollector getDeferringCollector() {
+        deferringCollector = new MergingBucketsDeferringCollector(context);
+        return deferringCollector;
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
+            final LeafBucketCollector sub) throws IOException {
+        if (valuesSource == null) {
+            return LeafBucketCollector.NO_OP_COLLECTOR;
+        }
+        final SortedNumericDocValues values = valuesSource.longValues(ctx);
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                assert bucket == 0;
+                if (values.advanceExact(doc)) {
+                    final int valuesCount = values.docValueCount();
+
+                    long previousRounded = Long.MIN_VALUE;
+                    for (int i = 0; i < valuesCount; ++i) {
+                        long value = values.nextValue();
+                        long rounded = roundingInfos[roundingIdx].rounding.round(value);
+                        assert rounded >= previousRounded;
+                        if (rounded == previousRounded) {
+                            continue;
+                        }
+                        long bucketOrd = bucketOrds.add(rounded);
+                        if (bucketOrd < 0) { // already seen
+                            bucketOrd = -1 - bucketOrd;
+                            collectExistingBucket(sub, doc, bucketOrd);
+                        } else {
+                            collectBucket(sub, doc, bucketOrd);
+                            while (roundingIdx < roundingInfos.length - 1
+                                    && bucketOrds.size() > (targetBuckets * roundingInfos[roundingIdx].getMaximumInnerInterval())) {
+                                increaseRounding();
+                            }
+                        }
+                        previousRounded = rounded;
+                    }
+                }
+            }
+
+            private void increaseRounding() {
+                try (LongHash oldBucketOrds = bucketOrds) {
+                    LongHash newBucketOrds = new LongHash(1, context.bigArrays());
+                    long[] mergeMap = new long[(int) oldBucketOrds.size()];
+                    Rounding newRounding = roundingInfos[++roundingIdx].rounding;
+                    for (int i = 0; i < oldBucketOrds.size(); i++) {
+                        long oldKey = oldBucketOrds.get(i);
+                        long newKey = newRounding.round(oldKey);
+                        long newBucketOrd = newBucketOrds.add(newKey);
+                        if (newBucketOrd >= 0) {
+                            mergeMap[i] = newBucketOrd;
+                        } else {
+                            mergeMap[i] = -1 - newBucketOrd;
+                        }
+                    }
+                    mergeBuckets(mergeMap, newBucketOrds.size());
+                    if (deferringCollector != null) {
+                        deferringCollector.mergeBuckets(mergeMap);
+                    }
+                    bucketOrds = newBucketOrds;
+                }
+            }
+        };
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
+        assert owningBucketOrdinal == 0;
+        consumeBucketsAndMaybeBreak((int) bucketOrds.size());
+
+        long[] bucketOrdArray = new long[(int) bucketOrds.size()];
+        for (int i = 0; i < bucketOrds.size(); i++) {
+            bucketOrdArray[i] = i;
+        }
+
+        runDeferredCollections(bucketOrdArray);
+
+        List<InternalAutoDateHistogram.Bucket> buckets = new ArrayList<>((int) bucketOrds.size());
+        for (long i = 0; i < bucketOrds.size(); i++) {
+            buckets.add(new InternalAutoDateHistogram.Bucket(bucketOrds.get(i), bucketDocCount(i), formatter, bucketAggregations(i)));
+        }
+
+        // the contract of the histogram aggregation is that shards must return
+        // buckets ordered by key in ascending order
+        CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator(this));
+
+        // value source will be null for unmapped fields
+        InternalAutoDateHistogram.BucketInfo emptyBucketInfo = new InternalAutoDateHistogram.BucketInfo(roundingInfos, roundingIdx,
+                buildEmptySubAggregations());
+
+        return new InternalAutoDateHistogram(name, buckets, targetBuckets, emptyBucketInfo, formatter, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        InternalAutoDateHistogram.BucketInfo emptyBucketInfo = new InternalAutoDateHistogram.BucketInfo(roundingInfos, roundingIdx,
+                buildEmptySubAggregations());
+        return new InternalAutoDateHistogram(name, Collections.emptyList(), targetBuckets, emptyBucketInfo, formatter,
+                pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public void doClose() {
+        Releasables.close(bucketOrds);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java
new file mode 100644
index 00000000000..051f2f9f6e7
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public final class AutoDateHistogramAggregatorFactory
+        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, AutoDateHistogramAggregatorFactory> {
+
+    private final int numBuckets;
+    private RoundingInfo[] roundingInfos;
+
+    public AutoDateHistogramAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, int numBuckets, RoundingInfo[] roundingInfos,
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            Map<String, Object> metaData) throws IOException {
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
+        this.numBuckets = numBuckets;
+        this.roundingInfos = roundingInfos;
+    }
+
+    @Override
+    protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, Aggregator parent, boolean collectsFromSingleBucket,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
+        if (collectsFromSingleBucket == false) {
+            return asMultiBucketAggregator(this, context, parent);
+        }
+        return createAggregator(valuesSource, parent, pipelineAggregators, metaData);
+    }
+
+    private Aggregator createAggregator(ValuesSource.Numeric valuesSource, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
+        return new AutoDateHistogramAggregator(name, factories, numBuckets, roundingInfos, valuesSource, config.format(), context, parent,
+                pipelineAggregators,
+                metaData);
+    }
+
+    @Override
+    protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+            throws IOException {
+        return createAggregator(null, parent, pipelineAggregators, metaData);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index e0b64d2cd5b..8b1f0c46421 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -28,13 +28,13 @@ import org.elasticsearch.common.util.LongHash;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalOrder;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.BucketOrder;
-import org.elasticsearch.search.aggregations.InternalOrder;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.internal.SearchContext;
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
new file mode 100644
index 00000000000..27c195cbdae
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
@@ -0,0 +1,601 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.apache.lucene.util.PriorityQueue;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.rounding.Rounding;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.KeyComparable;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Implementation of {@link Histogram}.
+ */
+public final class InternalAutoDateHistogram extends
+        InternalMultiBucketAggregation<InternalAutoDateHistogram, InternalAutoDateHistogram.Bucket> implements Histogram, HistogramFactory {
+
+    public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Histogram.Bucket, KeyComparable<Bucket> {
+
+        final long key;
+        final long docCount;
+        final InternalAggregations aggregations;
+        protected final transient DocValueFormat format;
+
+        public Bucket(long key, long docCount, DocValueFormat format,
+                InternalAggregations aggregations) {
+            this.format = format;
+            this.key = key;
+            this.docCount = docCount;
+            this.aggregations = aggregations;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public Bucket(StreamInput in, DocValueFormat format) throws IOException {
+            this.format = format;
+            key = in.readLong();
+            docCount = in.readVLong();
+            aggregations = InternalAggregations.readAggregations(in);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || obj.getClass() != InternalAutoDateHistogram.Bucket.class) {
+                return false;
+            }
+            InternalAutoDateHistogram.Bucket that = (InternalAutoDateHistogram.Bucket) obj;
+            // No need to take the keyed and format parameters into account,
+            // they are already stored and tested on the InternalDateHistogram object
+            return key == that.key
+                    && docCount == that.docCount
+                    && Objects.equals(aggregations, that.aggregations);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getClass(), key, docCount, aggregations);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeLong(key);
+            out.writeVLong(docCount);
+            aggregations.writeTo(out);
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return format.format(key).toString();
+        }
+
+        @Override
+        public Object getKey() {
+            return new DateTime(key, DateTimeZone.UTC);
+        }
+
+        @Override
+        public long getDocCount() {
+            return docCount;
+        }
+
+        @Override
+        public Aggregations getAggregations() {
+            return aggregations;
+        }
+
+        Bucket reduce(List<Bucket> buckets, Rounding rounding, ReduceContext context) {
+            List<InternalAggregations> aggregations = new ArrayList<>(buckets.size());
+            long docCount = 0;
+            for (Bucket bucket : buckets) {
+                docCount += bucket.docCount;
+                aggregations.add((InternalAggregations) bucket.getAggregations());
+            }
+            InternalAggregations aggs = InternalAggregations.reduce(aggregations, context);
+            return new InternalAutoDateHistogram.Bucket(rounding.round(key), docCount, format, aggs);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            String keyAsString = format.format(key).toString();
+            builder.startObject();
+            if (format != DocValueFormat.RAW) {
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), keyAsString);
+            }
+            builder.field(CommonFields.KEY.getPreferredName(), key);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int compareKey(Bucket other) {
+            return Long.compare(key, other.key);
+        }
+
+        public DocValueFormat getFormatter() {
+            return format;
+        }
+    }
+
+    static class BucketInfo {
+
+        final RoundingInfo[] roundingInfos;
+        final int roundingIdx;
+        final InternalAggregations emptySubAggregations;
+
+        BucketInfo(RoundingInfo[] roundings, int roundingIdx, InternalAggregations subAggregations) {
+            this.roundingInfos = roundings;
+            this.roundingIdx = roundingIdx;
+            this.emptySubAggregations = subAggregations;
+        }
+
+        BucketInfo(StreamInput in) throws IOException {
+            int size = in.readVInt();
+            roundingInfos = new RoundingInfo[size];
+            for (int i = 0; i < size; i++) {
+                roundingInfos[i] = new RoundingInfo(in);
+            }
+            roundingIdx = in.readVInt();
+            emptySubAggregations = InternalAggregations.readAggregations(in);
+        }
+
+        void writeTo(StreamOutput out) throws IOException {
+            out.writeVInt(roundingInfos.length);
+            for (RoundingInfo roundingInfo : roundingInfos) {
+                roundingInfo.writeTo(out);
+            }
+            out.writeVInt(roundingIdx);
+            emptySubAggregations.writeTo(out);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            BucketInfo that = (BucketInfo) obj;
+            return Objects.deepEquals(roundingInfos, that.roundingInfos)
+                    && Objects.equals(roundingIdx, that.roundingIdx)
+                    && Objects.equals(emptySubAggregations, that.emptySubAggregations);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getClass(), Arrays.hashCode(roundingInfos), roundingIdx, emptySubAggregations);
+        }
+    }
+
+    private final List<Bucket> buckets;
+    private final DocValueFormat format;
+    private final BucketInfo bucketInfo;
+    private final int targetBuckets;
+
+
+    InternalAutoDateHistogram(String name, List<Bucket> buckets, int targetBuckets, BucketInfo emptyBucketInfo, DocValueFormat formatter,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        super(name, pipelineAggregators, metaData);
+        this.buckets = buckets;
+        this.bucketInfo = emptyBucketInfo;
+        this.format = formatter;
+        this.targetBuckets = targetBuckets;
+    }
+
+    /**
+     * Stream from a stream.
+     */
+    public InternalAutoDateHistogram(StreamInput in) throws IOException {
+        super(in);
+        bucketInfo = new BucketInfo(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        buckets = in.readList(stream -> new Bucket(stream, format));
+        this.targetBuckets = in.readVInt();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        bucketInfo.writeTo(out);
+        out.writeNamedWriteable(format);
+        out.writeList(buckets);
+        out.writeVInt(targetBuckets);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return AutoDateHistogramAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<InternalAutoDateHistogram.Bucket> getBuckets() {
+        return Collections.unmodifiableList(buckets);
+    }
+
+    DocValueFormat getFormatter() {
+        return format;
+    }
+
+    public int getTargetBuckets() {
+        return targetBuckets;
+    }
+
+    public BucketInfo getBucketInfo() {
+        return bucketInfo;
+    }
+
+    @Override
+    public InternalAutoDateHistogram create(List<Bucket> buckets) {
+        return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators(), metaData);
+    }
+
+    @Override
+    public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+        return new Bucket(prototype.key, prototype.docCount, prototype.format, aggregations);
+    }
+
+    private static class IteratorAndCurrent {
+
+        private final Iterator<Bucket> iterator;
+        private Bucket current;
+
+        IteratorAndCurrent(Iterator<Bucket> iterator) {
+            this.iterator = iterator;
+            current = iterator.next();
+        }
+
+    }
+
+    /**
+     * This method works almost exactly the same as
+     * InternalDateHistogram#reduceBuckets(List, ReduceContext), the different
+     * here is that we need to round all the keys we see using the highest level
+     * rounding returned across all the shards so the resolution of the buckets
+     * is the same and they can be reduced together.
+     */
+    private BucketReduceResult reduceBuckets(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+
+        // First we need to find the highest level rounding used across all the
+        // shards
+        int reduceRoundingIdx = 0;
+        for (InternalAggregation aggregation : aggregations) {
+            int aggRoundingIdx = ((InternalAutoDateHistogram) aggregation).bucketInfo.roundingIdx;
+            if (aggRoundingIdx > reduceRoundingIdx) {
+                reduceRoundingIdx = aggRoundingIdx;
+            }
+        }
+        // This rounding will be used to reduce all the buckets
+        RoundingInfo reduceRoundingInfo = bucketInfo.roundingInfos[reduceRoundingIdx];
+        Rounding reduceRounding = reduceRoundingInfo.rounding;
+
+        final PriorityQueue<IteratorAndCurrent> pq = new PriorityQueue<IteratorAndCurrent>(aggregations.size()) {
+            @Override
+            protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurrent b) {
+                return a.current.key < b.current.key;
+            }
+        };
+        for (InternalAggregation aggregation : aggregations) {
+            InternalAutoDateHistogram histogram = (InternalAutoDateHistogram) aggregation;
+            if (histogram.buckets.isEmpty() == false) {
+                pq.add(new IteratorAndCurrent(histogram.buckets.iterator()));
+            }
+        }
+
+        List<Bucket> reducedBuckets = new ArrayList<>();
+        if (pq.size() > 0) {
+            // list of buckets coming from different shards that have the same key
+            List<Bucket> currentBuckets = new ArrayList<>();
+            double key = reduceRounding.round(pq.top().current.key);
+
+            do {
+                final IteratorAndCurrent top = pq.top();
+
+                if (reduceRounding.round(top.current.key) != key) {
+                    // the key changes, reduce what we already buffered and reset the buffer for current buckets
+                    final Bucket reduced = currentBuckets.get(0).reduce(currentBuckets, reduceRounding, reduceContext);
+                    reduceContext.consumeBucketsAndMaybeBreak(1);
+                    reducedBuckets.add(reduced);
+                    currentBuckets.clear();
+                    key = reduceRounding.round(top.current.key);
+                }
+
+                currentBuckets.add(top.current);
+
+                if (top.iterator.hasNext()) {
+                    final Bucket next = top.iterator.next();
+                    assert next.key > top.current.key : "shards must return data sorted by key";
+                    top.current = next;
+                    pq.updateTop();
+                } else {
+                    pq.pop();
+                }
+            } while (pq.size() > 0);
+
+            if (currentBuckets.isEmpty() == false) {
+                final Bucket reduced = currentBuckets.get(0).reduce(currentBuckets, reduceRounding, reduceContext);
+                reduceContext.consumeBucketsAndMaybeBreak(1);
+                reducedBuckets.add(reduced);
+            }
+        }
+
+        return mergeBucketsIfNeeded(reducedBuckets, reduceRoundingIdx, reduceRoundingInfo, reduceContext);
+    }
+
+    private BucketReduceResult mergeBucketsIfNeeded(List<Bucket> reducedBuckets, int reduceRoundingIdx, RoundingInfo reduceRoundingInfo,
+            ReduceContext reduceContext) {
+        while (reducedBuckets.size() > (targetBuckets * reduceRoundingInfo.getMaximumInnerInterval())
+                && reduceRoundingIdx < bucketInfo.roundingInfos.length - 1) {
+            reduceRoundingIdx++;
+            reduceRoundingInfo = bucketInfo.roundingInfos[reduceRoundingIdx];
+            reducedBuckets = mergeBuckets(reducedBuckets, reduceRoundingInfo.rounding, reduceContext);
+        }
+        return new BucketReduceResult(reducedBuckets, reduceRoundingInfo, reduceRoundingIdx);
+    }
+
+    private List<Bucket> mergeBuckets(List<Bucket> reducedBuckets, Rounding reduceRounding, ReduceContext reduceContext) {
+        List<Bucket> mergedBuckets = new ArrayList<>();
+
+        List<Bucket> sameKeyedBuckets = new ArrayList<>();
+        double key = Double.NaN;
+        for (Bucket bucket : reducedBuckets) {
+            long roundedBucketKey = reduceRounding.round(bucket.key);
+            if (Double.isNaN(key)) {
+                key = roundedBucketKey;
+                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(bucket) - 1);
+                sameKeyedBuckets.add(createBucket(key, bucket.docCount, bucket.aggregations));
+            } else if (roundedBucketKey == key) {
+                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(bucket) - 1);
+                sameKeyedBuckets.add(createBucket(key, bucket.docCount, bucket.aggregations));
+            } else {
+                reduceContext.consumeBucketsAndMaybeBreak(1);
+                mergedBuckets.add(sameKeyedBuckets.get(0).reduce(sameKeyedBuckets, reduceRounding, reduceContext));
+                sameKeyedBuckets.clear();
+                key = roundedBucketKey;
+                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(bucket) - 1);
+                sameKeyedBuckets.add(createBucket(key, bucket.docCount, bucket.aggregations));
+            }
+        }
+        if (sameKeyedBuckets.isEmpty() == false) {
+            reduceContext.consumeBucketsAndMaybeBreak(1);
+            mergedBuckets.add(sameKeyedBuckets.get(0).reduce(sameKeyedBuckets, reduceRounding, reduceContext));
+        }
+        reducedBuckets = mergedBuckets;
+        return reducedBuckets;
+    }
+
+    private static class BucketReduceResult {
+        List<Bucket> buckets;
+        RoundingInfo roundingInfo;
+        int roundingIdx;
+
+        BucketReduceResult(List<Bucket> buckets, RoundingInfo roundingInfo, int roundingIdx) {
+            this.buckets = buckets;
+            this.roundingInfo = roundingInfo;
+            this.roundingIdx = roundingIdx;
+
+        }
+    }
+
+    private BucketReduceResult addEmptyBuckets(BucketReduceResult currentResult, ReduceContext reduceContext) {
+        List<Bucket> list = currentResult.buckets;
+        if (list.isEmpty()) {
+            return currentResult;
+        }
+        int roundingIdx = getAppropriateRounding(list.get(0).key, list.get(list.size() - 1).key, currentResult.roundingIdx,
+                bucketInfo.roundingInfos);
+        RoundingInfo roundingInfo = bucketInfo.roundingInfos[roundingIdx];
+        Rounding rounding = roundingInfo.rounding;
+        // merge buckets using the new rounding
+        list = mergeBuckets(list, rounding, reduceContext);
+
+        Bucket lastBucket = null;
+        ListIterator<Bucket> iter = list.listIterator();
+        InternalAggregations reducedEmptySubAggs = InternalAggregations.reduce(Collections.singletonList(bucketInfo.emptySubAggregations),
+                reduceContext);
+
+        // Add the empty buckets within the data,
+        // e.g. if the data series is [1,2,3,7] there're 3 empty buckets that will be created for 4,5,6
+        while (iter.hasNext()) {
+            Bucket nextBucket = list.get(iter.nextIndex());
+            if (lastBucket != null) {
+                long key = rounding.nextRoundingValue(lastBucket.key);
+                while (key < nextBucket.key) {
+                    reduceContext.consumeBucketsAndMaybeBreak(1);
+                    iter.add(new InternalAutoDateHistogram.Bucket(key, 0, format, reducedEmptySubAggs));
+                    key = rounding.nextRoundingValue(key);
+                }
+                assert key == nextBucket.key : "key: " + key + ", nextBucket.key: " + nextBucket.key;
+            }
+            lastBucket = iter.next();
+        }
+        return new BucketReduceResult(list, roundingInfo, roundingIdx);
+    }
+
+    private int getAppropriateRounding(long minKey, long maxKey, int roundingIdx, RoundingInfo[] roundings) {
+        if (roundingIdx == roundings.length - 1) {
+            return roundingIdx;
+        }
+        int currentRoundingIdx = roundingIdx;
+
+        // Getting the accurate number of required buckets can be slow for large
+        // ranges at low roundings so get a rough estimate of the rounding first
+        // so we are at most 1 away from the correct rounding and then get the
+        // accurate rounding value
+        for (int i = currentRoundingIdx + 1; i < roundings.length; i++) {
+            long dataDuration = maxKey - minKey;
+            long roughEstimateRequiredBuckets = dataDuration / roundings[i].getRoughEstimateDurationMillis();
+            if (roughEstimateRequiredBuckets < targetBuckets * roundings[i].getMaximumInnerInterval()) {
+                currentRoundingIdx = i - 1;
+                break;
+            } else if (i == roundingIdx - 1) {
+                currentRoundingIdx = i;
+                break;
+            }
+        }
+
+        int requiredBuckets = 0;
+        do {
+            Rounding currentRounding = roundings[currentRoundingIdx].rounding;
+            long currentKey = minKey;
+            requiredBuckets = 0;
+            while (currentKey < maxKey) {
+                requiredBuckets++;
+                currentKey = currentRounding.nextRoundingValue(currentKey);
+            }
+            currentRoundingIdx++;
+        } while (requiredBuckets > (targetBuckets * roundings[roundingIdx].getMaximumInnerInterval())
+                && currentRoundingIdx < roundings.length);
+        // The loop will increase past the correct rounding index here so we
+        // need to subtract one to get the rounding index we need
+        return currentRoundingIdx - 1;
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        BucketReduceResult reducedBucketsResult = reduceBuckets(aggregations, reduceContext);
+
+        if (reduceContext.isFinalReduce()) {
+            // adding empty buckets if needed
+            reducedBucketsResult = addEmptyBuckets(reducedBucketsResult, reduceContext);
+
+            // Adding empty buckets may have tipped us over the target so merge the buckets again if needed
+            reducedBucketsResult = mergeBucketsIfNeeded(reducedBucketsResult.buckets, reducedBucketsResult.roundingIdx,
+                    reducedBucketsResult.roundingInfo, reduceContext);
+
+            // Now finally see if we need to merge consecutive buckets together to make a coarser interval at the same rounding
+            reducedBucketsResult = maybeMergeConsecutiveBuckets(reducedBucketsResult, reduceContext);
+        }
+
+        BucketInfo bucketInfo = new BucketInfo(this.bucketInfo.roundingInfos, reducedBucketsResult.roundingIdx,
+                this.bucketInfo.emptySubAggregations);
+
+        return new InternalAutoDateHistogram(getName(), reducedBucketsResult.buckets, targetBuckets, bucketInfo, format,
+                pipelineAggregators(), getMetaData());
+    }
+
+    private BucketReduceResult maybeMergeConsecutiveBuckets(BucketReduceResult reducedBucketsResult, ReduceContext reduceContext) {
+        List<Bucket> buckets = reducedBucketsResult.buckets;
+        RoundingInfo roundingInfo = reducedBucketsResult.roundingInfo;
+        int roundingIdx = reducedBucketsResult.roundingIdx;
+        if (buckets.size() > targetBuckets) {
+            for (int interval : roundingInfo.innerIntervals) {
+                int resultingBuckets = buckets.size() / interval;
+                if (resultingBuckets <= targetBuckets) {
+                    return mergeConsecutiveBuckets(buckets, interval, roundingIdx, roundingInfo, reduceContext);
+                }
+            }
+        }
+        return reducedBucketsResult;
+    }
+
+    private BucketReduceResult mergeConsecutiveBuckets(List<Bucket> reducedBuckets, int mergeInterval, int roundingIdx,
+            RoundingInfo roundingInfo, ReduceContext reduceContext) {
+        List<Bucket> mergedBuckets = new ArrayList<>();
+        List<Bucket> sameKeyedBuckets = new ArrayList<>();
+
+        double key = roundingInfo.rounding.round(reducedBuckets.get(0).key);
+        for (int i = 0; i < reducedBuckets.size(); i++) {
+            Bucket bucket = reducedBuckets.get(i);
+            if (i % mergeInterval == 0 && sameKeyedBuckets.isEmpty() == false) {
+                reduceContext.consumeBucketsAndMaybeBreak(1);
+                mergedBuckets.add(sameKeyedBuckets.get(0).reduce(sameKeyedBuckets, roundingInfo.rounding, reduceContext));
+                sameKeyedBuckets.clear();
+                key = roundingInfo.rounding.round(bucket.key);
+            }
+            reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(bucket) - 1);
+            sameKeyedBuckets.add(createBucket(key, bucket.docCount, bucket.aggregations));
+        }
+        if (sameKeyedBuckets.isEmpty() == false) {
+            reduceContext.consumeBucketsAndMaybeBreak(1);
+            mergedBuckets.add(sameKeyedBuckets.get(0).reduce(sameKeyedBuckets, roundingInfo.rounding, reduceContext));
+        }
+        return new BucketReduceResult(mergedBuckets, roundingInfo, roundingIdx);
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(CommonFields.BUCKETS.getPreferredName());
+        for (Bucket bucket : buckets) {
+            bucket.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    // HistogramFactory method impls
+
+    @Override
+    public Number getKey(MultiBucketsAggregation.Bucket bucket) {
+        return ((Bucket) bucket).key;
+    }
+
+    @Override
+    public Number nextKey(Number key) {
+        return bucketInfo.roundingInfos[bucketInfo.roundingIdx].rounding.nextRoundingValue(key.longValue());
+    }
+
+    @Override
+    public InternalAggregation createAggregation(List<MultiBucketsAggregation.Bucket> buckets) {
+        // convert buckets to the right type
+        List<Bucket> buckets2 = new ArrayList<>(buckets.size());
+        for (Object b : buckets) {
+            buckets2.add((Bucket) b);
+        }
+        buckets2 = Collections.unmodifiableList(buckets2);
+        return new InternalAutoDateHistogram(name, buckets2, targetBuckets, bucketInfo, format, pipelineAggregators(), getMetaData());
+    }
+
+    @Override
+    public Bucket createBucket(Number key, long docCount, InternalAggregations aggregations) {
+        return new Bucket(key.longValue(), docCount, format, aggregations);
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        InternalAutoDateHistogram that = (InternalAutoDateHistogram) obj;
+        return Objects.equals(buckets, that.buckets)
+                && Objects.equals(format, that.format)
+                && Objects.equals(bucketInfo, that.bucketInfo);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(buckets, format, bucketInfo);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
index 84dec2c983e..669bda5574d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
@@ -424,7 +424,7 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
                     iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                     key = nextKey(key).longValue();
                 }
-                assert key == nextBucket.key;
+                assert key == nextBucket.key : "key: " + key + ", nextBucket.key: " + nextBucket.key;
             }
             lastBucket = iter.next();
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedAutoDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedAutoDateHistogram.java
new file mode 100644
index 00000000000..caca44f9f2e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ParsedAutoDateHistogram.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.util.List;
+
+public class ParsedAutoDateHistogram extends ParsedMultiBucketAggregation<ParsedAutoDateHistogram.ParsedBucket> implements Histogram {
+
+    @Override
+    public String getType() {
+        return AutoDateHistogramAggregationBuilder.NAME;
+    }
+
+    @Override
+    public List<? extends Histogram.Bucket> getBuckets() {
+        return buckets;
+    }
+
+    private static ObjectParser<ParsedAutoDateHistogram, Void> PARSER =
+            new ObjectParser<>(ParsedAutoDateHistogram.class.getSimpleName(), true, ParsedAutoDateHistogram::new);
+    static {
+        declareMultiBucketAggregationFields(PARSER,
+                parser -> ParsedBucket.fromXContent(parser, false),
+                parser -> ParsedBucket.fromXContent(parser, true));
+    }
+
+    public static ParsedAutoDateHistogram fromXContent(XContentParser parser, String name) throws IOException {
+        ParsedAutoDateHistogram aggregation = PARSER.parse(parser, null);
+        aggregation.setName(name);
+        return aggregation;
+    }
+
+    public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket {
+
+        private Long key;
+
+        @Override
+        public Object getKey() {
+            if (key != null) {
+                return new DateTime(key, DateTimeZone.UTC);
+            }
+            return null;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            String keyAsString = super.getKeyAsString();
+            if (keyAsString != null) {
+                return keyAsString;
+            }
+            if (key != null) {
+                return Long.toString(key);
+            }
+            return null;
+        }
+
+        @Override
+        protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), key);
+        }
+
+        static ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException {
+            return parseXContent(parser, keyed, ParsedBucket::new, (p, bucket) -> bucket.key = p.longValue());
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
index 79984f58949..fcafce3936e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
@@ -37,6 +37,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilterTests;
 import org.elasticsearch.search.aggregations.bucket.filter.InternalFiltersTests;
 import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGridTests;
 import org.elasticsearch.search.aggregations.bucket.global.InternalGlobalTests;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalAutoDateHistogramTests;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogramTests;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogramTests;
 import org.elasticsearch.search.aggregations.bucket.missing.InternalMissingTests;
@@ -125,6 +126,7 @@ public class AggregationsTests extends ESTestCase {
         aggsTests.add(new InternalGeoCentroidTests());
         aggsTests.add(new InternalHistogramTests());
         aggsTests.add(new InternalDateHistogramTests());
+        aggsTests.add(new InternalAutoDateHistogramTests());
         aggsTests.add(new LongTermsTests());
         aggsTests.add(new DoubleTermsTests());
         aggsTests.add(new StringTermsTests());
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/AutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/AutoDateHistogramTests.java
new file mode 100644
index 00000000000..3a10edf1833
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/AutoDateHistogramTests.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
+
+public class AutoDateHistogramTests extends BaseAggregationTestCase<AutoDateHistogramAggregationBuilder> {
+
+    @Override
+    protected AutoDateHistogramAggregationBuilder createTestAggregatorBuilder() {
+        AutoDateHistogramAggregationBuilder builder = new AutoDateHistogramAggregationBuilder(randomAlphaOfLengthBetween(1, 10));
+        builder.field(INT_FIELD_NAME);
+        builder.setNumBuckets(randomIntBetween(1, 100000));
+        if (randomBoolean()) {
+            builder.format("###.##");
+        }
+        if (randomBoolean()) {
+            builder.missing(randomIntBetween(0, 10));
+        }
+        if (randomBoolean()) {
+            builder.timeZone(randomDateTimeZone());
+        }
+        return builder;
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java
new file mode 100644
index 00000000000..7cf29e3aa9c
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java
@@ -0,0 +1,1332 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
+import org.elasticsearch.search.aggregations.metrics.stats.Stats;
+import org.hamcrest.Matchers;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.chrono.ISOChronology;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Consumer;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class AutoDateHistogramAggregatorTests extends AggregatorTestCase {
+
+    private static final String DATE_FIELD = "date";
+    private static final String INSTANT_FIELD = "instant";
+
+    private static final List<String> dataset = Arrays.asList(
+            "2010-03-12T01:07:45",
+            "2010-04-27T03:43:34",
+            "2012-05-18T04:11:00",
+            "2013-05-29T05:11:31",
+            "2013-10-31T08:24:05",
+            "2015-02-13T13:09:32",
+            "2015-06-24T13:47:43",
+            "2015-11-13T16:14:34",
+            "2016-03-04T17:09:50",
+            "2017-12-12T22:55:46");
+
+    public void testMatchNoDocs() throws IOException {
+        testBothCases(new MatchNoDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD),
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+    }
+
+    public void testMatchAllDocs() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        testSearchCase(query, dataset,
+                aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD),
+                histogram -> assertEquals(10, histogram.getBuckets().size())
+        );
+        testSearchAndReduceCase(query, dataset,
+                aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD),
+                histogram -> assertEquals(8, histogram.getBuckets().size())
+        );
+    }
+
+    public void testSubAggregations() throws IOException {
+        Query query = new MatchAllDocsQuery();
+        testSearchAndReduceCase(query, dataset,
+                aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD)
+                        .subAggregation(AggregationBuilders.stats("stats").field(DATE_FIELD)),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(8, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2010-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+                    Stats stats = bucket.getAggregations().get("stats");
+                    assertEquals("2010-03-12T01:07:45.000Z", stats.getMinAsString());
+                    assertEquals("2010-04-27T03:43:34.000Z", stats.getMaxAsString());
+                    assertEquals(2L, stats.getCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2011-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+                    stats = bucket.getAggregations().get("stats");
+                    assertTrue(Double.isInfinite(stats.getMin()));
+                    assertTrue(Double.isInfinite(stats.getMax()));
+                    assertEquals(0L, stats.getCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2012-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                    stats = bucket.getAggregations().get("stats");
+                    assertEquals("2012-05-18T04:11:00.000Z", stats.getMinAsString());
+                    assertEquals("2012-05-18T04:11:00.000Z", stats.getMaxAsString());
+                    assertEquals(1L, stats.getCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2013-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+                    stats = bucket.getAggregations().get("stats");
+                    assertEquals("2013-05-29T05:11:31.000Z", stats.getMinAsString());
+                    assertEquals("2013-10-31T08:24:05.000Z", stats.getMaxAsString());
+                    assertEquals(2L, stats.getCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2014-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+                    stats = bucket.getAggregations().get("stats");
+                    assertTrue(Double.isInfinite(stats.getMin()));
+                    assertTrue(Double.isInfinite(stats.getMax()));
+                    assertEquals(0L, stats.getCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2015-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                    stats = bucket.getAggregations().get("stats");
+                    assertEquals("2015-02-13T13:09:32.000Z", stats.getMinAsString());
+                    assertEquals("2015-11-13T16:14:34.000Z", stats.getMaxAsString());
+                    assertEquals(3L, stats.getCount());
+
+                    bucket = buckets.get(6);
+                    assertEquals("2016-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                    stats = bucket.getAggregations().get("stats");
+                    assertEquals("2016-03-04T17:09:50.000Z", stats.getMinAsString());
+                    assertEquals("2016-03-04T17:09:50.000Z", stats.getMaxAsString());
+                    assertEquals(1L, stats.getCount());
+
+                    bucket = buckets.get(7);
+                    assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                    stats = bucket.getAggregations().get("stats");
+                    assertEquals("2017-12-12T22:55:46.000Z", stats.getMinAsString());
+                    assertEquals("2017-12-12T22:55:46.000Z", stats.getMaxAsString());
+                    assertEquals(1L, stats.getCount());
+                });
+    }
+
+    public void testNoDocs() throws IOException {
+        Query query = new MatchNoDocsQuery();
+        List<String> dates = Collections.emptyList();
+        Consumer<AutoDateHistogramAggregationBuilder> aggregation = agg -> agg.setNumBuckets(10).field(DATE_FIELD);
+
+        testSearchCase(query, dates, aggregation,
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+        testSearchAndReduceCase(query, dates, aggregation,
+                histogram -> assertNull(histogram)
+        );
+    }
+
+    public void testAggregateWrongField() throws IOException {
+        testBothCases(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(10).field("wrong_field"),
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+    }
+
+    public void testIntervalYear() throws IOException {
+        testSearchCase(LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2015-01-01"), asLong("2017-12-31")), dataset,
+                aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(5, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2015-02-13T13:09:32.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2015-06-24T13:47:43.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2015-11-13T16:14:34.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2016-03-04T17:09:50.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-12-12T22:55:46.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+        testSearchAndReduceCase(LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2015-01-01"), asLong("2017-12-31")), dataset,
+                aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2015-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2016-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalMonth() throws IOException {
+        testSearchCase(new MatchAllDocsQuery(),
+                Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"),
+                aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD), histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(6, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-02T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-03T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-03-04T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-03-05T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-03-06T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(),
+                Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"),
+                aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-03-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testWithLargeNumberOfBuckets() {
+        Query query = new MatchAllDocsQuery();
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> testSearchCase(query, dataset,
+                aggregation -> aggregation.setNumBuckets(MultiBucketConsumerService.DEFAULT_MAX_BUCKETS+1).field(DATE_FIELD),
+                // since an exception is thrown, this assertion won't be invoked.
+                histogram -> assertTrue(false)
+            ));
+        assertThat(exception.getMessage(), containsString("must be less than"));
+    }
+
+    public void testIntervalDay() throws IOException {
+        testSearchCase(new MatchAllDocsQuery(),
+                Arrays.asList("2017-02-01", "2017-02-02", "2017-02-02", "2017-02-03", "2017-02-03", "2017-02-03", "2017-02-05"),
+                aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD), histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(4, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-02T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-03T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-05T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01",
+                        "2017-02-02",
+                        "2017-02-02",
+                        "2017-02-03",
+                        "2017-02-03",
+                        "2017-02-03",
+                        "2017-02-05"
+                ),
+                aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(5, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-02T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-03T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-04T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-05T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalDayWithTZ() throws IOException {
+        testSearchCase(new MatchAllDocsQuery(),
+                Arrays.asList("2017-02-01", "2017-02-02", "2017-02-02", "2017-02-03", "2017-02-03", "2017-02-03", "2017-02-05"),
+                aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(4, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-01-31T23:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T23:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-02T23:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-04T23:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(),
+                Arrays.asList("2017-02-01", "2017-02-02", "2017-02-02", "2017-02-03", "2017-02-03", "2017-02-03", "2017-02-05"),
+                aggregation -> aggregation.setNumBuckets(5).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)), histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(5, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-01-31T00:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-02T00:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-03T00:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-04T00:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                });
+    }
+
+    public void testIntervalHour() throws IOException {
+        testSearchCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:00.000Z",
+                        "2017-02-01T09:35:00.000Z",
+                        "2017-02-01T10:15:00.000Z",
+                        "2017-02-01T13:06:00.000Z",
+                        "2017-02-01T14:04:00.000Z",
+                        "2017-02-01T14:05:00.000Z",
+                        "2017-02-01T15:59:00.000Z",
+                        "2017-02-01T16:06:00.000Z",
+                        "2017-02-01T16:48:00.000Z",
+                        "2017-02-01T16:59:00.000Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(10, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:02:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T09:35:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T10:15:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T13:06:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T14:04:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T14:05:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(6);
+                    assertEquals("2017-02-01T15:59:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(7);
+                    assertEquals("2017-02-01T16:06:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(8);
+                    assertEquals("2017-02-01T16:48:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(9);
+                    assertEquals("2017-02-01T16:59:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+        testSearchAndReduceCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:00.000Z",
+                        "2017-02-01T09:35:00.000Z",
+                        "2017-02-01T10:15:00.000Z",
+                        "2017-02-01T13:06:00.000Z",
+                        "2017-02-01T14:04:00.000Z",
+                        "2017-02-01T14:05:00.000Z",
+                        "2017-02-01T15:59:00.000Z",
+                        "2017-02-01T16:06:00.000Z",
+                        "2017-02-01T16:48:00.000Z",
+                        "2017-02-01T16:59:00.000Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(8, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T11:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T12:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(6);
+                    assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(7);
+                    assertEquals("2017-02-01T16:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalHourWithTZ() throws IOException {
+        testSearchCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:00.000Z",
+                        "2017-02-01T09:35:00.000Z",
+                        "2017-02-01T10:15:00.000Z",
+                        "2017-02-01T13:06:00.000Z",
+                        "2017-02-01T14:04:00.000Z",
+                        "2017-02-01T14:05:00.000Z",
+                        "2017-02-01T15:59:00.000Z",
+                        "2017-02-01T16:06:00.000Z",
+                        "2017-02-01T16:48:00.000Z",
+                        "2017-02-01T16:59:00.000Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(10, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T08:02:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T08:35:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T09:15:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T12:06:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T13:04:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T13:05:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(6);
+                    assertEquals("2017-02-01T14:59:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(7);
+                    assertEquals("2017-02-01T15:06:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(8);
+                    assertEquals("2017-02-01T15:48:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(9);
+                    assertEquals("2017-02-01T15:59:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+        testSearchAndReduceCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:00.000Z",
+                        "2017-02-01T09:35:00.000Z",
+                        "2017-02-01T10:15:00.000Z",
+                        "2017-02-01T13:06:00.000Z",
+                        "2017-02-01T14:04:00.000Z",
+                        "2017-02-01T14:05:00.000Z",
+                        "2017-02-01T15:59:00.000Z",
+                        "2017-02-01T16:06:00.000Z",
+                        "2017-02-01T16:48:00.000Z",
+                        "2017-02-01T16:59:00.000Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD).timeZone(DateTimeZone.forOffsetHours(-1)),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(8, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T08:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T09:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T10:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T11:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T12:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T13:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(6);
+                    assertEquals("2017-02-01T14:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(7);
+                    assertEquals("2017-02-01T15:00:00.000-01:00", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testAllSecondIntervals() throws IOException {
+        DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+        List<String> dataset = new ArrayList<>();
+        DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC());
+        for (int i = 0; i < 600; i++) {
+            DateTime date = startDate.plusSeconds(i);
+            dataset.add(format.print(date));
+        }
+
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(600, buckets.size());
+                    for (int i = 0; i < 600; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusSeconds(i), bucket.getKey());
+                        assertEquals(1, bucket.getDocCount());
+                    }
+                });
+
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(120, buckets.size());
+                    for (int i = 0; i < 120; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusSeconds(i * 5), bucket.getKey());
+                        assertEquals(5, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(100).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(60, buckets.size());
+                    for (int i = 0; i < 60; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusSeconds(i * 10), bucket.getKey());
+                        assertEquals(10, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(50).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(20, buckets.size());
+                    for (int i = 0; i < 20; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusSeconds(i * 30), bucket.getKey());
+                        assertEquals(30, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(10, buckets.size());
+                    for (int i = 0; i < 10; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusMinutes(i), bucket.getKey());
+                        assertEquals(60, bucket.getDocCount());
+                    }
+                });
+    }
+
+    public void testAllMinuteIntervals() throws IOException {
+        DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+        List<String> dataset = new ArrayList<>();
+        DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC());
+        for (int i = 0; i < 600; i++) {
+            DateTime date = startDate.plusMinutes(i);
+            dataset.add(format.print(date));
+        }
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(600, buckets.size());
+                    for (int i = 0; i < 600; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusMinutes(i), bucket.getKey());
+                        assertEquals(1, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(120, buckets.size());
+                    for (int i = 0; i < 120; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusMinutes(i * 5), bucket.getKey());
+                        assertEquals(5, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(100).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(60, buckets.size());
+                    for (int i = 0; i < 60; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusMinutes(i * 10), bucket.getKey());
+                        assertEquals(10, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(50).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(20, buckets.size());
+                    for (int i = 0; i < 20; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusMinutes(i * 30), bucket.getKey());
+                        assertEquals(30, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(10, buckets.size());
+                    for (int i = 0; i < 10; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusHours(i), bucket.getKey());
+                        assertEquals(60, bucket.getDocCount());
+                    }
+                });
+    }
+
+    public void testAllHourIntervals() throws IOException {
+        DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+        List<String> dataset = new ArrayList<>();
+        DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC());
+        for (int i = 0; i < 600; i++) {
+            DateTime date = startDate.plusHours(i);
+            dataset.add(format.print(date));
+        }
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(600, buckets.size());
+                    for (int i = 0; i < 600; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusHours(i), bucket.getKey());
+                        assertEquals(1, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(200, buckets.size());
+                    for (int i = 0; i < 200; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusHours(i * 3), bucket.getKey());
+                        assertEquals(3, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(100).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(50, buckets.size());
+                    for (int i = 0; i < 50; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusHours(i * 12), bucket.getKey());
+                        assertEquals(12, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(30).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(25, buckets.size());
+                    for (int i = 0; i < 25; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusDays(i), bucket.getKey());
+                        assertEquals(24, bucket.getDocCount());
+                    }
+                });
+    }
+
+    public void testAllDayIntervals() throws IOException {
+        DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+        List<String> dataset = new ArrayList<>();
+        DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC());
+        for (int i = 0; i < 700; i++) {
+            DateTime date = startDate.plusDays(i);
+            dataset.add(format.print(date));
+        }
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(700).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(700, buckets.size());
+                    for (int i = 0; i < 700; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusDays(i), bucket.getKey());
+                        assertEquals(1, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(100, buckets.size());
+                    for (int i = 0; i < 100; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusDays(i * 7), bucket.getKey());
+                        assertEquals(7, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(30).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(24, buckets.size());
+                    for (int i = 0; i < 24; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusMonths(i), bucket.getKey());
+                        assertThat(bucket.getDocCount(), Matchers.lessThanOrEqualTo(31L));
+                    }
+                });
+    }
+
+    public void testAllMonthIntervals() throws IOException {
+        DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+        List<String> dataset = new ArrayList<>();
+        DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC());
+        for (int i = 0; i < 600; i++) {
+            DateTime date = startDate.plusMonths(i);
+            dataset.add(format.print(date));
+        }
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(600, buckets.size());
+                    for (int i = 0; i < 600; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusMonths(i), bucket.getKey());
+                        assertEquals(1, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(200, buckets.size());
+                    for (int i = 0; i < 200; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusMonths(i * 3), bucket.getKey());
+                        assertEquals(3, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.setNumBuckets(60).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(50, buckets.size());
+                    for (int i = 0; i < 50; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusYears(i), bucket.getKey());
+                        assertEquals(12, bucket.getDocCount());
+                    }
+                });
+    }
+
+    public void testAllYearIntervals() throws IOException {
+        DateTimeFormatter format = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+        List<String> dataset = new ArrayList<>();
+        DateTime startDate = new DateTime(2017, 01, 01, 00, 00, 00, ISOChronology.getInstanceUTC());
+        for (int i = 0; i < 600; i++) {
+            DateTime date = startDate.plusYears(i);
+            dataset.add(format.print(date));
+        }
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(600).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(600, buckets.size());
+                    for (int i = 0; i < 600; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusYears(i), bucket.getKey());
+                        assertEquals(1, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(300).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(120, buckets.size());
+                    for (int i = 0; i < 120; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusYears(i * 5), bucket.getKey());
+                        assertEquals(5, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(100).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(60, buckets.size());
+                    for (int i = 0; i < 60; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusYears(i * 10), bucket.getKey());
+                        assertEquals(10, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(50).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(30, buckets.size());
+                    for (int i = 0; i < 30; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusYears(i * 20), bucket.getKey());
+                        assertEquals(20, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(20).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(12, buckets.size());
+                    for (int i = 0; i < 12; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusYears(i * 50), bucket.getKey());
+                        assertEquals(50, bucket.getDocCount());
+                    }
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(), dataset, aggregation -> aggregation.setNumBuckets(10).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(6, buckets.size());
+                    for (int i = 0; i < 6; i++) {
+                        Histogram.Bucket bucket = buckets.get(i);
+                        assertEquals(startDate.plusYears(i * 100), bucket.getKey());
+                        assertEquals(100, bucket.getDocCount());
+                    }
+                });
+    }
+
+    public void testInterval3Hour() throws IOException {
+        testSearchCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:00.000Z",
+                        "2017-02-01T09:35:00.000Z",
+                        "2017-02-01T10:15:00.000Z",
+                        "2017-02-01T13:06:00.000Z",
+                        "2017-02-01T14:04:00.000Z",
+                        "2017-02-01T14:05:00.000Z",
+                        "2017-02-01T15:59:00.000Z",
+                        "2017-02-01T16:06:00.000Z",
+                        "2017-02-01T16:48:00.000Z",
+                        "2017-02-01T16:59:00.000Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(8).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(10, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:02:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T09:35:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T10:15:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T13:06:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T14:04:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T14:05:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(6);
+                    assertEquals("2017-02-01T15:59:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(7);
+                    assertEquals("2017-02-01T16:06:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(8);
+                    assertEquals("2017-02-01T16:48:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(9);
+                    assertEquals("2017-02-01T16:59:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+        testSearchAndReduceCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:00.000Z",
+                        "2017-02-01T09:35:00.000Z",
+                        "2017-02-01T10:15:00.000Z",
+                        "2017-02-01T13:06:00.000Z",
+                        "2017-02-01T14:04:00.000Z",
+                        "2017-02-01T14:05:00.000Z",
+                        "2017-02-01T15:59:00.000Z",
+                        "2017-02-01T16:06:00.000Z",
+                        "2017-02-01T16:48:00.000Z",
+                        "2017-02-01T16:59:00.000Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(6).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T12:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(4, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalMinute() throws IOException {
+        testSearchCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:35.000Z",
+                        "2017-02-01T09:02:59.000Z",
+                        "2017-02-01T09:15:37.000Z",
+                        "2017-02-01T09:16:04.000Z",
+                        "2017-02-01T09:16:42.000Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(4).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(5, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:02:35.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T09:02:59.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T09:15:37.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T09:16:04.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T09:16:42.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+        testSearchAndReduceCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:35.000Z",
+                        "2017-02-01T09:02:59.000Z",
+                        "2017-02-01T09:15:37.000Z",
+                        "2017-02-01T09:16:04.000Z",
+                        "2017-02-01T09:16:42.000Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(15).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(15, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:02:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T09:03:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T09:04:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T09:05:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T09:06:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T09:07:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(6);
+                    assertEquals("2017-02-01T09:08:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(7);
+                    assertEquals("2017-02-01T09:09:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(8);
+                    assertEquals("2017-02-01T09:10:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(9);
+                    assertEquals("2017-02-01T09:11:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(10);
+                    assertEquals("2017-02-01T09:12:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(11);
+                    assertEquals("2017-02-01T09:13:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(12);
+                    assertEquals("2017-02-01T09:14:00.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(13);
+                    assertEquals("2017-02-01T09:15:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(14);
+                    assertEquals("2017-02-01T09:16:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalSecond() throws IOException {
+        testSearchCase(new MatchAllDocsQuery(),
+                Arrays.asList("2017-02-01T00:00:05.015Z", "2017-02-01T00:00:07.299Z", "2017-02-01T00:00:07.074Z",
+                        "2017-02-01T00:00:11.688Z", "2017-02-01T00:00:11.210Z", "2017-02-01T00:00:11.380Z"),
+                aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD), histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:05.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:07.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T00:00:11.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                });
+        testSearchAndReduceCase(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T00:00:05.015Z",
+                        "2017-02-01T00:00:07.299Z",
+                        "2017-02-01T00:00:07.074Z",
+                        "2017-02-01T00:00:11.688Z",
+                        "2017-02-01T00:00:11.210Z",
+                        "2017-02-01T00:00:11.380Z"
+                ),
+                aggregation -> aggregation.setNumBuckets(7).field(DATE_FIELD),
+                histogram -> {
+                    List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(7, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:05.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:06.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T00:00:07.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T00:00:08.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T00:00:09.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T00:00:10.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(6);
+                    assertEquals("2017-02-01T00:00:11.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    private void testSearchCase(Query query, List<String> dataset,
+            Consumer<AutoDateHistogramAggregationBuilder> configure,
+                                Consumer<Histogram> verify) throws IOException {
+        executeTestCase(false, query, dataset, configure, verify);
+    }
+
+    private void testSearchAndReduceCase(Query query, List<String> dataset,
+            Consumer<AutoDateHistogramAggregationBuilder> configure,
+                                         Consumer<Histogram> verify) throws IOException {
+        executeTestCase(true, query, dataset, configure, verify);
+    }
+
+    private void testBothCases(Query query, List<String> dataset,
+            Consumer<AutoDateHistogramAggregationBuilder> configure,
+                               Consumer<Histogram> verify) throws IOException {
+        testSearchCase(query, dataset, configure, verify);
+        testSearchAndReduceCase(query, dataset, configure, verify);
+    }
+
+    @Override
+    protected IndexSettings createIndexSettings() {
+        Settings nodeSettings = Settings.builder()
+            .put("search.max_buckets", 100000).build();
+        return new IndexSettings(
+            IndexMetaData.builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .creationDate(System.currentTimeMillis())
+                .build(),
+            nodeSettings
+        );
+    }
+
+    private void executeTestCase(boolean reduced, Query query, List<String> dataset,
+                                 Consumer<AutoDateHistogramAggregationBuilder> configure,
+                                 Consumer<Histogram> verify) throws IOException {
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                for (String date : dataset) {
+                    if (frequently()) {
+                        indexWriter.commit();
+                    }
+
+                    long instant = asLong(date);
+                    document.add(new SortedNumericDocValuesField(DATE_FIELD, instant));
+                    document.add(new LongPoint(INSTANT_FIELD, instant));
+                    indexWriter.addDocument(document);
+                    document.clear();
+                }
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                AutoDateHistogramAggregationBuilder aggregationBuilder = new AutoDateHistogramAggregationBuilder("_name");
+                if (configure != null) {
+                    configure.accept(aggregationBuilder);
+                }
+
+                DateFieldMapper.Builder builder = new DateFieldMapper.Builder("_name");
+                DateFieldMapper.DateFieldType fieldType = builder.fieldType();
+                fieldType.setHasDocValues(true);
+                fieldType.setName(aggregationBuilder.field());
+
+                InternalAutoDateHistogram histogram;
+                if (reduced) {
+                    histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType);
+                } else {
+                    histogram = search(indexSearcher, query, aggregationBuilder, fieldType);
+                }
+                verify.accept(histogram);
+            }
+        }
+    }
+
+    private static long asLong(String dateTime) {
+        return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(dateTime).getMillis();
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
new file mode 100644
index 00000000000..389371efd79
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.rounding.DateTimeUnit;
+import org.elasticsearch.common.rounding.Rounding;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalAutoDateHistogram.BucketInfo;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalMultiBucketAggregationTestCase;
+import org.joda.time.DateTime;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueHours;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+
+public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregationTestCase<InternalAutoDateHistogram> {
+
+    private DocValueFormat format;
+    private RoundingInfo[] roundingInfos;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        format = randomNumericDocValueFormat();
+
+        roundingInfos = new RoundingInfo[6];
+        roundingInfos[0] = new RoundingInfo(Rounding.builder(DateTimeUnit.SECOND_OF_MINUTE).build(), 1, 5, 10, 30);
+        roundingInfos[1] = new RoundingInfo(Rounding.builder(DateTimeUnit.MINUTES_OF_HOUR).build(), 1, 5, 10, 30);
+        roundingInfos[2] = new RoundingInfo(Rounding.builder(DateTimeUnit.HOUR_OF_DAY).build(), 1, 3, 12);
+        roundingInfos[3] = new RoundingInfo(Rounding.builder(DateTimeUnit.DAY_OF_MONTH).build(), 1, 7);
+        roundingInfos[4] = new RoundingInfo(Rounding.builder(DateTimeUnit.MONTH_OF_YEAR).build(), 1, 3);
+        roundingInfos[5] = new RoundingInfo(Rounding.builder(DateTimeUnit.YEAR_OF_CENTURY).build(), 1, 10, 20, 50, 100);
+    }
+
+    @Override
+    protected InternalAutoDateHistogram createTestInstance(String name,
+                                                       List<PipelineAggregator> pipelineAggregators,
+                                                       Map<String, Object> metaData,
+                                                       InternalAggregations aggregations) {
+        int nbBuckets = randomNumberOfBuckets();
+        int targetBuckets = randomIntBetween(1, nbBuckets * 2 + 1);
+        List<InternalAutoDateHistogram.Bucket> buckets = new ArrayList<>(nbBuckets);
+        long startingDate = System.currentTimeMillis();
+
+        long interval = randomIntBetween(1, 3);
+        long intervalMillis = randomFrom(timeValueSeconds(interval), timeValueMinutes(interval), timeValueHours(interval)).getMillis();
+
+        for (int i = 0; i < nbBuckets; i++) {
+            long key = startingDate + (intervalMillis * i);
+            buckets.add(i, new InternalAutoDateHistogram.Bucket(key, randomIntBetween(1, 100), format, aggregations));
+        }
+        InternalAggregations subAggregations = new InternalAggregations(Collections.emptyList());
+        BucketInfo bucketInfo = new BucketInfo(roundingInfos, randomIntBetween(0, roundingInfos.length - 1), subAggregations);
+
+        return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertReduced(InternalAutoDateHistogram reduced, List<InternalAutoDateHistogram> inputs) {
+        int roundingIdx = 0;
+        for (InternalAutoDateHistogram histogram : inputs) {
+            if (histogram.getBucketInfo().roundingIdx > roundingIdx) {
+                roundingIdx = histogram.getBucketInfo().roundingIdx;
+            }
+        }
+        Map<Long, Long> expectedCounts = new TreeMap<>();
+        for (Histogram histogram : inputs) {
+            for (Histogram.Bucket bucket : histogram.getBuckets()) {
+                expectedCounts.compute(roundingInfos[roundingIdx].rounding.round(((DateTime) bucket.getKey()).getMillis()),
+                        (key, oldValue) -> (oldValue == null ? 0 : oldValue) + bucket.getDocCount());
+            }
+        }
+        Map<Long, Long> actualCounts = new TreeMap<>();
+        for (Histogram.Bucket bucket : reduced.getBuckets()) {
+            actualCounts.compute(((DateTime) bucket.getKey()).getMillis(),
+                    (key, oldValue) -> (oldValue == null ? 0 : oldValue) + bucket.getDocCount());
+        }
+        assertEquals(expectedCounts, actualCounts);
+    }
+
+    @Override
+    protected Writeable.Reader<InternalAutoDateHistogram> instanceReader() {
+        return InternalAutoDateHistogram::new;
+    }
+
+    @Override
+    protected Class<? extends ParsedMultiBucketAggregation> implementationClass() {
+        return ParsedAutoDateHistogram.class;
+    }
+
+    @Override
+    protected InternalAutoDateHistogram mutateInstance(InternalAutoDateHistogram instance) {
+        String name = instance.getName();
+        List<InternalAutoDateHistogram.Bucket> buckets = instance.getBuckets();
+        int targetBuckets = instance.getTargetBuckets();
+        BucketInfo bucketInfo = instance.getBucketInfo();
+        List<PipelineAggregator> pipelineAggregators = instance.pipelineAggregators();
+        Map<String, Object> metaData = instance.getMetaData();
+        switch (between(0, 3)) {
+        case 0:
+            name += randomAlphaOfLength(5);
+            break;
+        case 1:
+            buckets = new ArrayList<>(buckets);
+            buckets.add(new InternalAutoDateHistogram.Bucket(randomNonNegativeLong(), randomIntBetween(1, 100), format,
+                    InternalAggregations.EMPTY));
+            break;
+        case 2:
+            int roundingIdx = bucketInfo.roundingIdx == bucketInfo.roundingInfos.length - 1 ? 0 : bucketInfo.roundingIdx + 1;
+            bucketInfo = new BucketInfo(bucketInfo.roundingInfos, roundingIdx, bucketInfo.emptySubAggregations);
+            break;
+        case 3:
+            if (metaData == null) {
+                metaData = new HashMap<>(1);
+            } else {
+                metaData = new HashMap<>(instance.getMetaData());
+            }
+            metaData.put(randomAlphaOfLength(15), randomInt());
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators, metaData);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 07f25986655..e84f2a99a11 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -87,7 +87,6 @@ import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
-import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 
 /**
  * Base class for testing {@link Aggregator} implementations.
@@ -229,7 +228,7 @@ public abstract class AggregatorTestCase extends ESTestCase {
         });
         when(searchContext.bitsetFilterCache()).thenReturn(new BitsetFilterCache(indexSettings, mock(Listener.class)));
         doAnswer(invocation -> {
-            /* Store the releasables so we can release them at the end of the test case. This is important because aggregations don't
+            /* Store the release-ables so we can release them at the end of the test case. This is important because aggregations don't
              * close their sub-aggregations. This is fairly similar to what the production code does. */
             releasables.add((Releasable) invocation.getArguments()[0]);
             return null;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
index 838b0e315ea..15e44853a97 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -53,8 +53,10 @@ import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBu
 import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedAutoDateHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram;
 import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
@@ -181,6 +183,7 @@ public abstract class InternalAggregationTestCase<T extends InternalAggregation>
         map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
         map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
         map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));
+        map.put(AutoDateHistogramAggregationBuilder.NAME, (p, c) -> ParsedAutoDateHistogram.fromXContent(p, (String) c));
         map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
         map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
         map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalMultiBucketAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalMultiBucketAggregationTestCase.java
index 952b6c02794..6f0aebe2396 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalMultiBucketAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalMultiBucketAggregationTestCase.java
@@ -149,7 +149,8 @@ public abstract class InternalMultiBucketAggregationTestCase<T extends InternalA
     protected void assertMultiBucketsAggregation(MultiBucketsAggregation expected, MultiBucketsAggregation actual, boolean checkOrder) {
         Class<? extends ParsedMultiBucketAggregation> parsedClass = implementationClass();
         assertNotNull("Parsed aggregation class must not be null", parsedClass);
-        assertTrue(parsedClass.isInstance(actual));
+        assertTrue("Unexpected parsed class, expected instance of: " + actual + ", but was: " + parsedClass,
+                parsedClass.isInstance(actual));
 
         assertTrue(expected instanceof InternalAggregation);
         assertEquals(expected.getName(), actual.getName());

From b1bf643e41b2bf2bc1360887b4768836c9159214 Mon Sep 17 00:00:00 2001
From: Vladimir Dolzhenko <vladimir.dolzhenko@gmail.com>
Date: Fri, 13 Jul 2018 20:05:49 +0200
Subject: [PATCH 014/260] lazy snapshot repository initialization (#31606)

lazy snapshot repository initialization
---
 .../repositories/url/URLRepository.java       |  33 +++--
 .../repositories/url/URLRepositoryTests.java  |  51 ++++++-
 .../repositories/azure/AzureRepository.java   |  42 ++++--
 .../azure/AzureRepositorySettingsTests.java   |  29 ++--
 .../gcs/GoogleCloudStorageRepository.java     |  24 ++--
 ...eCloudStorageBlobStoreRepositoryTests.java |  10 +-
 .../repositories/hdfs/HdfsRepository.java     |  26 ++--
 .../repositories/s3/S3Repository.java         |  71 +++++----
 .../repositories/s3/S3RepositoryPlugin.java   |   4 +-
 .../s3/RepositoryCredentialsTests.java        |  26 +++-
 .../s3/S3BlobStoreRepositoryTests.java        |  15 +-
 .../repositories/s3/S3RepositoryTests.java    |  38 +++--
 .../repositories/RepositoriesService.java     |  73 ++++++----
 .../VerifyNodeRepositoryAction.java           |   2 +-
 .../blobstore/BlobStoreRepository.java        | 136 ++++++++++++++----
 .../repositories/fs/FsRepository.java         |  32 +++--
 ...ClusterStateServiceRandomUpdatesTests.java |   2 +-
 .../BlobStoreRepositoryRestoreTests.java      |  11 +-
 .../blobstore/BlobStoreRepositoryTests.java   |  35 ++++-
 .../fs}/FsBlobStoreRepositoryIT.java          |  11 +-
 .../SharedClusterSnapshotRestoreIT.java       |  23 ++-
 .../snapshots/mockstore/MockRepository.java   |  13 +-
 .../ESBlobStoreRepositoryIntegTestCase.java   |  56 ++++++--
 23 files changed, 543 insertions(+), 220 deletions(-)
 rename server/src/test/java/org/elasticsearch/{snapshots => repositories/fs}/FsBlobStoreRepositoryIT.java (79%)

diff --git a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
index c1128fd683a..98b8c0a1945 100644
--- a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.repositories.url;
 
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.url.URLBlobStore;
@@ -31,7 +32,6 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
-import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -71,33 +71,44 @@ public class URLRepository extends BlobStoreRepository {
 
     private final Environment environment;
 
-    private final URLBlobStore blobStore;
-
     private final BlobPath basePath;
 
+    private final URL url;
+
     /**
      * Constructs a read-only URL-based repository
      */
     public URLRepository(RepositoryMetaData metadata, Environment environment,
-                         NamedXContentRegistry namedXContentRegistry) throws IOException {
+                         NamedXContentRegistry namedXContentRegistry) {
         super(metadata, environment.settings(), namedXContentRegistry);
 
         if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) ==  false) {
             throw new RepositoryException(metadata.name(), "missing url");
         }
+        this.environment = environment;
         supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(settings);
         urlWhiteList = ALLOWED_URLS_SETTING.get(settings).toArray(new URIPattern[]{});
-        this.environment = environment;
-
-        URL url = URL_SETTING.exists(metadata.settings()) ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
-        URL normalizedURL = checkURL(url);
-        blobStore = new URLBlobStore(settings, normalizedURL);
         basePath = BlobPath.cleanPath();
+        url = URL_SETTING.exists(metadata.settings())
+            ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
     }
 
     @Override
-    protected BlobStore blobStore() {
-        return blobStore;
+    protected BlobStore createBlobStore() {
+        URL normalizedURL = checkURL(url);
+        return new URLBlobStore(settings, normalizedURL);
+    }
+
+    // only use for testing
+    @Override
+    protected BlobContainer blobContainer() {
+        return super.blobContainer();
+    }
+
+    // only use for testing
+    @Override
+    protected BlobStore getBlobStore() {
+        return super.getBlobStore();
     }
 
     @Override
diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java
index 1af4c1eaba9..2de4c132673 100644
--- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java
+++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java
@@ -31,8 +31,22 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collections;
 
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
+
 public class URLRepositoryTests extends ESTestCase {
 
+    private URLRepository createRepository(Settings baseSettings, RepositoryMetaData repositoryMetaData) {
+        return new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
+            new NamedXContentRegistry(Collections.emptyList())) {
+            @Override
+            protected void assertSnapshotOrGenericThread() {
+                // eliminate thread name check as we create repo manually on test/main threads
+            }
+        };
+    }
+
     public void testWhiteListingRepoURL() throws IOException {
         String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
         Settings baseSettings = Settings.builder()
@@ -41,8 +55,12 @@ public class URLRepositoryTests extends ESTestCase {
             .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
             .build();
         RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
-        new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
-            new NamedXContentRegistry(Collections.emptyList()));
+        final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
+        repository.start();
+
+        assertThat("blob store has to be lazy initialized", repository.getBlobStore(), is(nullValue()));
+        repository.blobContainer();
+        assertThat("blobContainer has to initialize blob store", repository.getBlobStore(), not(nullValue()));
     }
 
     public void testIfNotWhiteListedMustSetRepoURL() throws IOException {
@@ -52,9 +70,10 @@ public class URLRepositoryTests extends ESTestCase {
             .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
             .build();
         RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
+        repository.start();
         try {
-            new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
-                new NamedXContentRegistry(Collections.emptyList()));
+            repository.blobContainer();
             fail("RepositoryException should have been thrown.");
         } catch (RepositoryException e) {
             String msg = "[url] file url [" + repoPath
@@ -73,13 +92,33 @@ public class URLRepositoryTests extends ESTestCase {
             .put(URLRepository.SUPPORTED_PROTOCOLS_SETTING.getKey(), "http,https")
             .build();
         RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
+        repository.start();
         try {
-            new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
-                new NamedXContentRegistry(Collections.emptyList()));
+            repository.blobContainer();
             fail("RepositoryException should have been thrown.");
         } catch (RepositoryException e) {
             assertEquals("[url] unsupported url protocol [file] from URL [" + repoPath +"]", e.getMessage());
         }
     }
 
+    public void testNonNormalizedUrl() throws IOException {
+        Settings baseSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(URLRepository.ALLOWED_URLS_SETTING.getKey(), "file:/tmp/")
+            .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), "file:/var/" )
+            .build();
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
+        repository.start();
+        try {
+            repository.blobContainer();
+            fail("RepositoryException should have been thrown.");
+        } catch (RepositoryException e) {
+            assertEquals("[url] file url [file:/var/] doesn't match any of the locations "
+                + "specified by path.repo or repositories.url.allowed_urls",
+                e.getMessage());
+        }
+    }
+
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 47b398a4c2f..0797c78af33 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -38,7 +38,6 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 import org.elasticsearch.snapshots.SnapshotCreationException;
 import org.elasticsearch.snapshots.SnapshotId;
 
-import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.List;
 import java.util.Locale;
@@ -78,25 +77,21 @@ public class AzureRepository extends BlobStoreRepository {
         public static final Setting<Boolean> READONLY_SETTING = Setting.boolSetting("readonly", false, Property.NodeScope);
     }
 
-    private final AzureBlobStore blobStore;
     private final BlobPath basePath;
     private final ByteSizeValue chunkSize;
     private final boolean compress;
+    private final Environment environment;
+    private final AzureStorageService storageService;
     private final boolean readonly;
 
     public AzureRepository(RepositoryMetaData metadata, Environment environment, NamedXContentRegistry namedXContentRegistry,
-            AzureStorageService storageService) throws IOException, URISyntaxException, StorageException {
+            AzureStorageService storageService) {
         super(metadata, environment.settings(), namedXContentRegistry);
-        this.blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
         this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings());
         this.compress = Repository.COMPRESS_SETTING.get(metadata.settings());
-        // If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
-        // For secondary_only setting, the repository should be read only
-        if (Repository.READONLY_SETTING.exists(metadata.settings())) {
-            this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
-        } else {
-            this.readonly = this.blobStore.getLocationMode() == LocationMode.SECONDARY_ONLY;
-        }
+        this.environment = environment;
+        this.storageService = storageService;
+
         final String basePath = Strings.trimLeadingCharacter(Repository.BASE_PATH_SETTING.get(metadata.settings()), '/');
         if (Strings.hasLength(basePath)) {
             // Remove starting / if any
@@ -108,15 +103,33 @@ public class AzureRepository extends BlobStoreRepository {
         } else {
             this.basePath = BlobPath.cleanPath();
         }
-        logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
-                "using container [{}], chunk_size [{}], compress [{}], base_path [{}]", blobStore, chunkSize, compress, basePath));
+
+        // If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
+        // For secondary_only setting, the repository should be read only
+        final LocationMode locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
+        if (Repository.READONLY_SETTING.exists(metadata.settings())) {
+            this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
+        } else {
+            this.readonly = locationMode == LocationMode.SECONDARY_ONLY;
+        }
+    }
+
+    // only use for testing
+    @Override
+    protected BlobStore getBlobStore() {
+        return super.getBlobStore();
     }
 
     /**
      * {@inheritDoc}
      */
     @Override
-    protected BlobStore blobStore() {
+    protected AzureBlobStore createBlobStore() throws URISyntaxException, StorageException {
+        final AzureBlobStore blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
+
+        logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
+            "using container [{}], chunk_size [{}], compress [{}], base_path [{}]",
+            blobStore, chunkSize, compress, basePath));
         return blobStore;
     }
 
@@ -144,6 +157,7 @@ public class AzureRepository extends BlobStoreRepository {
     @Override
     public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData clusterMetadata) {
         try {
+            final AzureBlobStore blobStore = (AzureBlobStore) blobStore();
             if (blobStore.containerExist() == false) {
                 throw new IllegalArgumentException("The bucket [" + blobStore + "] does not exist. Please create it before "
                         + " creating an azure snapshot repository backed by it.");
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
index 639905042cf..b4b71577cbc 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.repositories.azure;
 
 import com.microsoft.azure.storage.LocationMode;
-import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -30,76 +29,76 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-import java.net.URISyntaxException;
-
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 
 public class AzureRepositorySettingsTests extends ESTestCase {
 
-    private AzureRepository azureRepository(Settings settings) throws StorageException, IOException, URISyntaxException {
+    private AzureRepository azureRepository(Settings settings) {
         Settings internalSettings = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath())
             .putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths())
             .put(settings)
             .build();
-        return new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings),
+        final AzureRepository azureRepository = new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings),
             TestEnvironment.newEnvironment(internalSettings), NamedXContentRegistry.EMPTY, mock(AzureStorageService.class));
+        assertThat(azureRepository.getBlobStore(), is(nullValue()));
+        return azureRepository;
     }
 
-    public void testReadonlyDefault() throws StorageException, IOException, URISyntaxException {
+    public void testReadonlyDefault() {
         assertThat(azureRepository(Settings.EMPTY).isReadOnly(), is(false));
     }
 
-    public void testReadonlyDefaultAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
+    public void testReadonlyDefaultAndReadonlyOn() {
         assertThat(azureRepository(Settings.builder()
             .put("readonly", true)
             .build()).isReadOnly(), is(true));
     }
 
-    public void testReadonlyWithPrimaryOnly() throws StorageException, IOException, URISyntaxException {
+    public void testReadonlyWithPrimaryOnly() {
         assertThat(azureRepository(Settings.builder()
             .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_ONLY.name())
             .build()).isReadOnly(), is(false));
     }
 
-    public void testReadonlyWithPrimaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
+    public void testReadonlyWithPrimaryOnlyAndReadonlyOn() {
         assertThat(azureRepository(Settings.builder()
             .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_ONLY.name())
             .put("readonly", true)
             .build()).isReadOnly(), is(true));
     }
 
-    public void testReadonlyWithSecondaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
+    public void testReadonlyWithSecondaryOnlyAndReadonlyOn() {
         assertThat(azureRepository(Settings.builder()
             .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.SECONDARY_ONLY.name())
             .put("readonly", true)
             .build()).isReadOnly(), is(true));
     }
 
-    public void testReadonlyWithSecondaryOnlyAndReadonlyOff() throws StorageException, IOException, URISyntaxException {
+    public void testReadonlyWithSecondaryOnlyAndReadonlyOff() {
         assertThat(azureRepository(Settings.builder()
             .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.SECONDARY_ONLY.name())
             .put("readonly", false)
             .build()).isReadOnly(), is(false));
     }
 
-    public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
+    public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOn() {
         assertThat(azureRepository(Settings.builder()
             .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_THEN_SECONDARY.name())
             .put("readonly", true)
             .build()).isReadOnly(), is(true));
     }
 
-    public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOff() throws StorageException, IOException, URISyntaxException {
+    public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOff() {
         assertThat(azureRepository(Settings.builder()
             .put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_THEN_SECONDARY.name())
             .put("readonly", false)
             .build()).isReadOnly(), is(false));
     }
 
-    public void testChunkSize() throws StorageException, IOException, URISyntaxException {
+    public void testChunkSize() {
         // default chunk size
         AzureRepository azureRepository = azureRepository(Settings.EMPTY);
         assertEquals(AzureStorageService.MAX_CHUNK_SIZE, azureRepository.chunkSize());
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
index 83d48eeda20..fe6c8889bd2 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -22,7 +22,6 @@ package org.elasticsearch.repositories.gcs;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
-import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -56,18 +55,19 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
             byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope, Property.Dynamic);
     static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
 
-    private final ByteSizeValue chunkSize;
-    private final boolean compress;
+    private final GoogleCloudStorageService storageService;
     private final BlobPath basePath;
-    private final GoogleCloudStorageBlobStore blobStore;
+    private final boolean compress;
+    private final ByteSizeValue chunkSize;
+    private final String bucket;
+    private final String clientName;
 
     GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment environment,
                                         NamedXContentRegistry namedXContentRegistry,
-                                        GoogleCloudStorageService storageService) throws Exception {
+                                        GoogleCloudStorageService storageService) {
         super(metadata, environment.settings(), namedXContentRegistry);
+        this.storageService = storageService;
 
-        String bucket = getSetting(BUCKET, metadata);
-        String clientName = CLIENT_NAME.get(metadata.settings());
         String basePath = BASE_PATH.get(metadata.settings());
         if (Strings.hasLength(basePath)) {
             BlobPath path = new BlobPath();
@@ -81,16 +81,14 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
 
         this.compress = getSetting(COMPRESS, metadata);
         this.chunkSize = getSetting(CHUNK_SIZE, metadata);
-
+        this.bucket = getSetting(BUCKET, metadata);
+        this.clientName = CLIENT_NAME.get(metadata.settings());
         logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}]", bucket, basePath, chunkSize, compress);
-
-        this.blobStore = new GoogleCloudStorageBlobStore(settings, bucket, clientName, storageService);
     }
 
-
     @Override
-    protected BlobStore blobStore() {
-        return blobStore;
+    protected GoogleCloudStorageBlobStore createBlobStore() {
+        return new GoogleCloudStorageBlobStore(settings, bucket, clientName, storageService);
     }
 
     @Override
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index 3692b26f2bb..6d5c1bbf853 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase;
 import org.junit.AfterClass;
 
@@ -34,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCase {
 
@@ -49,9 +51,10 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos
     }
 
     @Override
-    protected void createTestRepository(String name) {
+    protected void createTestRepository(String name, boolean verify) {
         assertAcked(client().admin().cluster().preparePutRepository(name)
                 .setType(GoogleCloudStorageRepository.TYPE)
+                .setVerify(verify)
                 .setSettings(Settings.builder()
                         .put("bucket", BUCKET)
                         .put("base_path", GoogleCloudStorageBlobStoreRepositoryTests.class.getSimpleName())
@@ -59,6 +62,11 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos
                         .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
     }
 
+    @Override
+    protected void afterCreationCheck(Repository repository) {
+        assertThat(repository, instanceOf(GoogleCloudStorageRepository.class));
+    }
+
     @AfterClass
     public static void wipeRepository() {
         blobs.clear();
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
index 5ef1c7d18d6..97285f9cecb 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
@@ -42,7 +42,6 @@ import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.blobstore.BlobPath;
-import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -61,29 +60,26 @@ public final class HdfsRepository extends BlobStoreRepository {
     private final ByteSizeValue chunkSize;
     private final boolean compress;
     private final BlobPath basePath = BlobPath.cleanPath();
-
-    private HdfsBlobStore blobStore;
+    private final URI uri;
+    private final String pathSetting;
 
     // buffer size passed to HDFS read/write methods
     // TODO: why 100KB?
     private static final ByteSizeValue DEFAULT_BUFFER_SIZE = new ByteSizeValue(100, ByteSizeUnit.KB);
 
     public HdfsRepository(RepositoryMetaData metadata, Environment environment,
-                          NamedXContentRegistry namedXContentRegistry) throws IOException {
+                          NamedXContentRegistry namedXContentRegistry) {
         super(metadata, environment.settings(), namedXContentRegistry);
 
         this.environment = environment;
         this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null);
         this.compress = metadata.settings().getAsBoolean("compress", false);
-    }
 
-    @Override
-    protected void doStart() {
         String uriSetting = getMetadata().settings().get("uri");
         if (Strings.hasText(uriSetting) == false) {
             throw new IllegalArgumentException("No 'uri' defined for hdfs snapshot/restore");
         }
-        URI uri = URI.create(uriSetting);
+        uri = URI.create(uriSetting);
         if ("hdfs".equalsIgnoreCase(uri.getScheme()) == false) {
             throw new IllegalArgumentException(String.format(Locale.ROOT,
                 "Invalid scheme [%s] specified in uri [%s]; only 'hdfs' uri allowed for hdfs snapshot/restore", uri.getScheme(), uriSetting));
@@ -93,16 +89,11 @@ public final class HdfsRepository extends BlobStoreRepository {
                 "Use 'path' option to specify a path [%s], not the uri [%s] for hdfs snapshot/restore", uri.getPath(), uriSetting));
         }
 
-        String pathSetting = getMetadata().settings().get("path");
+        pathSetting = getMetadata().settings().get("path");
         // get configuration
         if (pathSetting == null) {
             throw new IllegalArgumentException("No 'path' defined for hdfs snapshot/restore");
         }
-
-        // initialize our blobstore using elevated privileges.
-        SpecialPermission.check();
-        blobStore = AccessController.doPrivileged((PrivilegedAction<HdfsBlobStore>) () -> createBlobstore(uri, pathSetting, getMetadata().settings()));
-        super.doStart();
     }
 
     private HdfsBlobStore createBlobstore(URI uri, String path, Settings repositorySettings)  {
@@ -229,7 +220,12 @@ public final class HdfsRepository extends BlobStoreRepository {
     }
 
     @Override
-    protected BlobStore blobStore() {
+    protected HdfsBlobStore createBlobStore() {
+        // initialize our blobstore using elevated privileges.
+        SpecialPermission.check();
+        final HdfsBlobStore blobStore =
+            AccessController.doPrivileged((PrivilegedAction<HdfsBlobStore>)
+                () -> createBlobstore(uri, pathSetting, getMetadata().settings()));
         return blobStore;
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index f6f949aa4d0..ec60536f135 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -35,7 +35,6 @@ import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -144,30 +143,43 @@ class S3Repository extends BlobStoreRepository {
      */
     static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path");
 
-    private final S3BlobStore blobStore;
+    private final S3Service service;
 
-    private final BlobPath basePath;
+    private final String bucket;
+
+    private final ByteSizeValue bufferSize;
 
     private final ByteSizeValue chunkSize;
 
     private final boolean compress;
 
+    private final BlobPath basePath;
+
+    private final boolean serverSideEncryption;
+
+    private final String storageClass;
+
+    private final String cannedACL;
+
+    private final String clientName;
+
     /**
      * Constructs an s3 backed repository
      */
     S3Repository(final RepositoryMetaData metadata,
                  final Settings settings,
                  final NamedXContentRegistry namedXContentRegistry,
-                 final S3Service service) throws IOException {
+                 final S3Service service) {
         super(metadata, settings, namedXContentRegistry);
+        this.service = service;
 
-        final String bucket = BUCKET_SETTING.get(metadata.settings());
+        // Parse and validate the user's S3 Storage Class setting
+        this.bucket = BUCKET_SETTING.get(metadata.settings());
         if (bucket == null) {
             throw new RepositoryException(metadata.name(), "No bucket defined for s3 repository");
         }
 
-        final boolean serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings());
-        final ByteSizeValue bufferSize = BUFFER_SIZE_SETTING.get(metadata.settings());
+        this.bufferSize = BUFFER_SIZE_SETTING.get(metadata.settings());
         this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings());
         this.compress = COMPRESS_SETTING.get(metadata.settings());
 
@@ -177,33 +189,44 @@ class S3Repository extends BlobStoreRepository {
                 ") can't be lower than " + BUFFER_SIZE_SETTING.getKey() + " (" + bufferSize + ").");
         }
 
-        // Parse and validate the user's S3 Storage Class setting
-        final String storageClass = STORAGE_CLASS_SETTING.get(metadata.settings());
-        final String cannedACL = CANNED_ACL_SETTING.get(metadata.settings());
-        final String clientName = CLIENT_NAME.get(metadata.settings());
-
-        logger.debug("using bucket [{}], chunk_size [{}], server_side_encryption [{}], " +
-            "buffer_size [{}], cannedACL [{}], storageClass [{}]",
-            bucket, chunkSize, serverSideEncryption, bufferSize, cannedACL, storageClass);
-
-        // deprecated behavior: override client credentials from the cluster state
-        // (repository settings)
-        if (S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) {
-            overrideCredentialsFromClusterState(service);
-        }
-        blobStore = new S3BlobStore(settings, service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass);
-
         final String basePath = BASE_PATH_SETTING.get(metadata.settings());
         if (Strings.hasLength(basePath)) {
             this.basePath = new BlobPath().add(basePath);
         } else {
             this.basePath = BlobPath.cleanPath();
         }
+
+        this.serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings());
+
+        this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings());
+        this.cannedACL = CANNED_ACL_SETTING.get(metadata.settings());
+        this.clientName = CLIENT_NAME.get(metadata.settings());
+
+        logger.debug("using bucket [{}], chunk_size [{}], server_side_encryption [{}], " +
+                "buffer_size [{}], cannedACL [{}], storageClass [{}]",
+            bucket, chunkSize, serverSideEncryption, bufferSize, cannedACL, storageClass);
+
+        // (repository settings)
+        if (S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) {
+            overrideCredentialsFromClusterState(service);
+        }
     }
 
+    @Override
+    protected S3BlobStore createBlobStore() {
+        return new S3BlobStore(settings, service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass);
+    }
+
+    // only use for testing
     @Override
     protected BlobStore blobStore() {
-        return blobStore;
+        return super.blobStore();
+    }
+
+    // only use for testing
+    @Override
+    protected BlobStore getBlobStore() {
+        return super.getBlobStore();
     }
 
     @Override
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
index 79a5187059f..da3219f2aef 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
@@ -61,7 +61,7 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
         });
     }
 
-    private final S3Service service;
+    protected final S3Service service;
 
     public S3RepositoryPlugin(final Settings settings) {
         this(settings, new S3Service(settings));
@@ -77,7 +77,7 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
     // proxy method for testing
     protected S3Repository createRepository(final RepositoryMetaData metadata,
                                             final Settings settings,
-                                            final NamedXContentRegistry registry) throws IOException {
+                                            final NamedXContentRegistry registry) {
         return new S3Repository(metadata, settings, registry, service);
     }
 
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
index 744a27dc48e..7eb603b4b78 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
@@ -80,6 +80,16 @@ public class RepositoryCredentialsTests extends ESTestCase {
         ProxyS3RepositoryPlugin(Settings settings) {
             super(settings, new ProxyS3Service(settings));
         }
+
+        @Override
+        protected S3Repository createRepository(RepositoryMetaData metadata, Settings settings, NamedXContentRegistry registry) {
+            return new S3Repository(metadata, settings, registry, service){
+                @Override
+                protected void assertSnapshotOrGenericThread() {
+                    // eliminate thread name check as we create repo manually on test/main threads
+                }
+            };
+        }
     }
 
     public void testRepositoryCredentialsOverrideSecureCredentials() throws IOException {
@@ -102,8 +112,8 @@ public class RepositoryCredentialsTests extends ESTestCase {
                 .put(S3Repository.ACCESS_KEY_SETTING.getKey(), "insecure_aws_key")
                 .put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret").build());
         try (S3RepositoryPlugin s3Plugin = new ProxyS3RepositoryPlugin(settings);
-                S3Repository s3repo = s3Plugin.createRepository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY);
-                AmazonS3Reference s3Ref = ((S3BlobStore) s3repo.blobStore()).clientReference()) {
+             S3Repository s3repo = createAndStartRepository(metadata, s3Plugin);
+             AmazonS3Reference s3Ref = ((S3BlobStore) s3repo.blobStore()).clientReference()) {
             final AWSCredentials credentials = ((ProxyS3RepositoryPlugin.ClientAndCredentials) s3Ref.client()).credentials.getCredentials();
             assertThat(credentials.getAWSAccessKeyId(), is("insecure_aws_key"));
             assertThat(credentials.getAWSSecretKey(), is("insecure_aws_secret"));
@@ -125,8 +135,8 @@ public class RepositoryCredentialsTests extends ESTestCase {
                         .put(S3Repository.SECRET_KEY_SETTING.getKey(), "insecure_aws_secret")
                         .build());
         try (S3RepositoryPlugin s3Plugin = new ProxyS3RepositoryPlugin(Settings.EMPTY);
-                S3Repository s3repo = s3Plugin.createRepository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY);
-                AmazonS3Reference s3Ref = ((S3BlobStore) s3repo.blobStore()).clientReference()) {
+             S3Repository s3repo = createAndStartRepository(metadata, s3Plugin);
+             AmazonS3Reference s3Ref = ((S3BlobStore) s3repo.blobStore()).clientReference()) {
             final AWSCredentials credentials = ((ProxyS3RepositoryPlugin.ClientAndCredentials) s3Ref.client()).credentials.getCredentials();
             assertThat(credentials.getAWSAccessKeyId(), is("insecure_aws_key"));
             assertThat(credentials.getAWSSecretKey(), is("insecure_aws_secret"));
@@ -140,6 +150,12 @@ public class RepositoryCredentialsTests extends ESTestCase {
                         + " See the breaking changes documentation for the next major version.");
     }
 
+    private S3Repository createAndStartRepository(RepositoryMetaData metadata, S3RepositoryPlugin s3Plugin) {
+        final S3Repository repository = s3Plugin.createRepository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY);
+        repository.start();
+        return repository;
+    }
+
     public void testReinitSecureCredentials() throws IOException {
         final String clientName = randomFrom("default", "some_client");
         // initial client node settings
@@ -156,7 +172,7 @@ public class RepositoryCredentialsTests extends ESTestCase {
         }
         final RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", builder.build());
         try (S3RepositoryPlugin s3Plugin = new ProxyS3RepositoryPlugin(settings);
-                S3Repository s3repo = s3Plugin.createRepository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY)) {
+                S3Repository s3repo = createAndStartRepository(metadata, s3Plugin)) {
             try (AmazonS3Reference s3Ref = ((S3BlobStore) s3repo.blobStore()).clientReference()) {
                 final AWSCredentials credentials = ((ProxyS3RepositoryPlugin.ClientAndCredentials) s3Ref.client()).credentials
                         .getCredentials();
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
index b061e8e45ed..51fc48dfb59 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -51,6 +51,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.mock;
 
@@ -84,8 +85,11 @@ public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCa
     }
 
     @Override
-    protected void createTestRepository(final String name) {
-        assertAcked(client().admin().cluster().preparePutRepository(name).setType(S3Repository.TYPE).setSettings(Settings.builder()
+    protected void createTestRepository(final String name, boolean verify) {
+        assertAcked(client().admin().cluster().preparePutRepository(name)
+            .setType(S3Repository.TYPE)
+            .setVerify(verify)
+            .setSettings(Settings.builder()
                 .put(S3Repository.BUCKET_SETTING.getKey(), bucket)
                 .put(S3Repository.CLIENT_NAME.getKey(), client)
                 .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), bufferSize)
@@ -96,6 +100,11 @@ public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCa
                 .put(S3Repository.SECRET_KEY_SETTING.getKey(), "not_used_but_this_is_a_secret")));
     }
 
+    @Override
+    protected void afterCreationCheck(Repository repository) {
+        assertThat(repository, instanceOf(S3Repository.class));
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singletonList(TestS3RepositoryPlugin.class);
@@ -125,7 +134,7 @@ public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCa
 
     public void testInsecureRepositoryCredentials() throws Exception {
         final String repositoryName = "testInsecureRepositoryCredentials";
-        createTestRepository(repositoryName);
+        createAndCheckTestRepository(repositoryName);
         final NodeClient nodeClient = internalCluster().getInstance(NodeClient.class);
         final RestGetRepositoriesAction getRepoAction = new RestGetRepositoriesAction(Settings.EMPTY, mock(RestController.class),
                 internalCluster().getInstance(SettingsFilter.class));
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
index 14f53ae5d33..dcc46661bef 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
@@ -29,11 +29,13 @@ import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 
 public class S3RepositoryTests extends ESTestCase {
 
@@ -70,27 +72,27 @@ public class S3RepositoryTests extends ESTestCase {
         }
     }
 
-    public void testInvalidChunkBufferSizeSettings() throws IOException {
+    public void testInvalidChunkBufferSizeSettings() {
         // chunk < buffer should fail
         final Settings s1 = bufferAndChunkSettings(10, 5);
         final Exception e1 = expectThrows(RepositoryException.class,
-                () -> new S3Repository(getRepositoryMetaData(s1), Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service()));
+                () -> createS3Repo(getRepositoryMetaData(s1)));
         assertThat(e1.getMessage(), containsString("chunk_size (5mb) can't be lower than buffer_size (10mb)"));
         // chunk > buffer should pass
         final Settings s2 = bufferAndChunkSettings(5, 10);
-        new S3Repository(getRepositoryMetaData(s2), Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service()).close();
+        createS3Repo(getRepositoryMetaData(s2)).close();
         // chunk = buffer should pass
         final Settings s3 = bufferAndChunkSettings(5, 5);
-        new S3Repository(getRepositoryMetaData(s3), Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service()).close();
+        createS3Repo(getRepositoryMetaData(s3)).close();
         // buffer < 5mb should fail
         final Settings s4 = bufferAndChunkSettings(4, 10);
         final IllegalArgumentException e2 = expectThrows(IllegalArgumentException.class,
-                () -> new S3Repository(getRepositoryMetaData(s4), Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service())
+                () -> createS3Repo(getRepositoryMetaData(s4))
                         .close());
         assertThat(e2.getMessage(), containsString("failed to parse value [4mb] for setting [buffer_size], must be >= [5mb]"));
         final Settings s5 = bufferAndChunkSettings(5, 6000000);
         final IllegalArgumentException e3 = expectThrows(IllegalArgumentException.class,
-                () -> new S3Repository(getRepositoryMetaData(s5), Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service())
+                () -> createS3Repo(getRepositoryMetaData(s5))
                         .close());
         assertThat(e3.getMessage(), containsString("failed to parse value [6000000mb] for setting [chunk_size], must be <= [5tb]"));
     }
@@ -106,20 +108,32 @@ public class S3RepositoryTests extends ESTestCase {
         return new RepositoryMetaData("dummy-repo", "mock", Settings.builder().put(settings).build());
     }
 
-    public void testBasePathSetting() throws IOException {
+    public void testBasePathSetting() {
         final RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
                 .put(S3Repository.BASE_PATH_SETTING.getKey(), "foo/bar").build());
-        try (S3Repository s3repo = new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service())) {
+        try (S3Repository s3repo = createS3Repo(metadata)) {
             assertEquals("foo/bar/", s3repo.basePath().buildAsString());
         }
     }
 
-    public void testDefaultBufferSize() throws IOException {
+    public void testDefaultBufferSize() {
         final RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.EMPTY);
-        try (S3Repository s3repo = new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service())) {
-            final long defaultBufferSize = ((S3BlobStore) s3repo.blobStore()).bufferSizeInBytes();
+        try (S3Repository s3repo = createS3Repo(metadata)) {
+            assertThat(s3repo.getBlobStore(), is(nullValue()));
+            s3repo.start();
+            final long defaultBufferSize = ((S3BlobStore)s3repo.blobStore()).bufferSizeInBytes();
+            assertThat(s3repo.getBlobStore(), not(nullValue()));
             assertThat(defaultBufferSize, Matchers.lessThanOrEqualTo(100L * 1024 * 1024));
             assertThat(defaultBufferSize, Matchers.greaterThanOrEqualTo(5L * 1024 * 1024));
         }
     }
+
+    private S3Repository createS3Repo(RepositoryMetaData metadata) {
+        return new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service()) {
+            @Override
+            protected void assertSnapshotOrGenericThread() {
+                // eliminate thread name check as we create repo manually on test/main threads
+            }
+        };
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index d5b2a6413e9..c6cbaa50cdf 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.snapshots.SnapshotsService;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -58,16 +59,20 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
 
     private final ClusterService clusterService;
 
+    private final ThreadPool threadPool;
+
     private final VerifyNodeRepositoryAction verifyAction;
 
     private volatile Map<String, Repository> repositories = Collections.emptyMap();
 
     @Inject
     public RepositoriesService(Settings settings, ClusterService clusterService, TransportService transportService,
-                               Map<String, Repository.Factory> typesRegistry) {
+                               Map<String, Repository.Factory> typesRegistry,
+                               ThreadPool threadPool) {
         super(settings);
         this.typesRegistry = typesRegistry;
         this.clusterService = clusterService;
+        this.threadPool = threadPool;
         // Doesn't make sense to maintain repositories on non-master and non-data nodes
         // Nothing happens there anyway
         if (DiscoveryNode.isDataNode(settings) || DiscoveryNode.isMasterNode(settings)) {
@@ -208,39 +213,51 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
     public void verifyRepository(final String repositoryName, final ActionListener<VerifyResponse> listener) {
         final Repository repository = repository(repositoryName);
         try {
-            final String verificationToken = repository.startVerification();
-            if (verificationToken != null) {
+            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
                 try {
-                    verifyAction.verify(repositoryName, verificationToken, new ActionListener<VerifyResponse>() {
-                        @Override
-                        public void onResponse(VerifyResponse verifyResponse) {
-                            try {
-                                repository.endVerification(verificationToken);
-                            } catch (Exception e) {
-                                logger.warn(() -> new ParameterizedMessage("[{}] failed to finish repository verification", repositoryName), e);
-                                listener.onFailure(e);
-                                return;
-                            }
-                            listener.onResponse(verifyResponse);
-                        }
+                    final String verificationToken = repository.startVerification();
+                    if (verificationToken != null) {
+                        try {
+                            verifyAction.verify(repositoryName, verificationToken, new ActionListener<VerifyResponse>() {
+                                @Override
+                                public void onResponse(VerifyResponse verifyResponse) {
+                                    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
+                                        try {
+                                            repository.endVerification(verificationToken);
+                                        } catch (Exception e) {
+                                            logger.warn(() -> new ParameterizedMessage(
+                                                "[{}] failed to finish repository verification", repositoryName), e);
+                                            listener.onFailure(e);
+                                            return;
+                                        }
+                                        listener.onResponse(verifyResponse);
+                                    });
+                                }
 
-                        @Override
-                        public void onFailure(Exception e) {
-                            listener.onFailure(e);
+                                @Override
+                                public void onFailure(Exception e) {
+                                    listener.onFailure(e);
+                                }
+                            });
+                        } catch (Exception e) {
+                            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
+                                try {
+                                    repository.endVerification(verificationToken);
+                                } catch (Exception inner) {
+                                    inner.addSuppressed(e);
+                                    logger.warn(() -> new ParameterizedMessage(
+                                        "[{}] failed to finish repository verification", repositoryName), inner);
+                                }
+                                listener.onFailure(e);
+                            });
                         }
-                    });
-                } catch (Exception e) {
-                    try {
-                        repository.endVerification(verificationToken);
-                    } catch (Exception inner) {
-                        inner.addSuppressed(e);
-                        logger.warn(() -> new ParameterizedMessage("[{}] failed to finish repository verification", repositoryName), inner);
+                    } else {
+                        listener.onResponse(new VerifyResponse(new DiscoveryNode[0], new VerificationFailure[0]));
                     }
+                } catch (Exception e) {
                     listener.onFailure(e);
                 }
-            } else {
-                listener.onResponse(new VerifyResponse(new DiscoveryNode[0], new VerificationFailure[0]));
-            }
+            });
         } catch (Exception e) {
             listener.onFailure(e);
         }
diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
index 380ae974080..fbaf369912e 100644
--- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
@@ -61,7 +61,7 @@ public class VerifyNodeRepositoryAction  extends AbstractComponent {
         this.transportService = transportService;
         this.clusterService = clusterService;
         this.repositoriesService = repositoriesService;
-        transportService.registerRequestHandler(ACTION_NAME, VerifyNodeRepositoryRequest::new, ThreadPool.Names.SAME, new VerifyNodeRepositoryRequestHandler());
+        transportService.registerRequestHandler(ACTION_NAME, VerifyNodeRepositoryRequest::new, ThreadPool.Names.SNAPSHOT, new VerifyNodeRepositoryRequestHandler());
     }
 
     public void verify(String repository, String verificationToken, final ActionListener<VerifyResponse> listener) {
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 86131fe468d..22743e38839 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -34,6 +34,7 @@ import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.RateLimiter;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
@@ -102,6 +103,7 @@ import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.FilterInputStream;
 import java.io.IOException;
@@ -126,8 +128,8 @@ import static org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSna
 /**
  * BlobStore - based implementation of Snapshot Repository
  * <p>
- * This repository works with any {@link BlobStore} implementation. The blobStore should be initialized in the derived
- * class before {@link #doStart()} is called.
+ * This repository works with any {@link BlobStore} implementation. The blobStore could be (and preferred) lazy initialized in
+ * {@link #createBlobStore()}.
  * <p>
  * BlobStoreRepository maintains the following structure in the blob store
  * <pre>
@@ -169,8 +171,6 @@ import static org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSna
  */
 public abstract class BlobStoreRepository extends AbstractLifecycleComponent implements Repository {
 
-    private BlobContainer snapshotsBlobContainer;
-
     protected final RepositoryMetaData metadata;
 
     protected final NamedXContentRegistry namedXContentRegistry;
@@ -225,6 +225,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshots> indexShardSnapshotsFormat;
 
+    private final Object lock = new Object();
+
+    private final SetOnce<BlobContainer> blobContainer = new SetOnce<>();
+
+    private final SetOnce<BlobStore> blobStore = new SetOnce<>();
+
     /**
      * Constructs new BlobStoreRepository
      *
@@ -251,7 +257,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     @Override
     protected void doStart() {
-        this.snapshotsBlobContainer = blobStore().blobContainer(basePath());
         globalMetaDataFormat = new ChecksumBlobStoreFormat<>(METADATA_CODEC, METADATA_NAME_FORMAT,
             MetaData::fromXContent, namedXContentRegistry, isCompress());
         indexMetaDataFormat = new ChecksumBlobStoreFormat<>(INDEX_METADATA_CODEC, METADATA_NAME_FORMAT,
@@ -265,17 +270,82 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     @Override
     protected void doClose() {
-        try {
-            blobStore().close();
-        } catch (Exception t) {
-            logger.warn("cannot close blob store", t);
+        BlobStore store;
+        // to close blobStore if blobStore initialization is started during close
+        synchronized (lock) {
+            store = blobStore.get();
+        }
+        if (store != null) {
+            try {
+                store.close();
+            } catch (Exception t) {
+                logger.warn("cannot close blob store", t);
+            }
         }
     }
 
+    // package private, only use for testing
+    BlobContainer getBlobContainer() {
+        return blobContainer.get();
+    }
+
+    // for test purposes only
+    protected BlobStore getBlobStore() {
+        return blobStore.get();
+    }
+
     /**
-     * Returns the BlobStore to read and write data.
+     * maintains single lazy instance of {@link BlobContainer}
      */
-    protected abstract BlobStore blobStore();
+    protected BlobContainer blobContainer() {
+        assertSnapshotOrGenericThread();
+
+        BlobContainer blobContainer = this.blobContainer.get();
+        if (blobContainer == null) {
+           synchronized (lock) {
+               blobContainer = this.blobContainer.get();
+               if (blobContainer == null) {
+                   blobContainer = blobStore().blobContainer(basePath());
+                   this.blobContainer.set(blobContainer);
+               }
+           }
+        }
+
+        return blobContainer;
+    }
+
+    /**
+     * maintains single lazy instance of {@link BlobStore}
+     */
+    protected BlobStore blobStore() {
+        assertSnapshotOrGenericThread();
+
+        BlobStore store = blobStore.get();
+        if (store == null) {
+            synchronized (lock) {
+                store = blobStore.get();
+                if (store == null) {
+                    if (lifecycle.started() == false) {
+                        throw new RepositoryException(metadata.name(), "repository is not in started state");
+                    }
+                    try {
+                        store = createBlobStore();
+                    } catch (RepositoryException e) {
+                        throw e;
+                    } catch (Exception e) {
+                        throw new RepositoryException(metadata.name(), "cannot create blob store" , e);
+                    }
+                    blobStore.set(store);
+                }
+            }
+        }
+        return store;
+    }
+
+    /**
+     * Creates new BlobStore to read and write data.
+     */
+    protected abstract BlobStore createBlobStore() throws Exception;
 
     /**
      * Returns base path of the repository
@@ -319,12 +389,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             if (repositoryData.getAllSnapshotIds().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
                 throw new InvalidSnapshotNameException(metadata.name(), snapshotId.getName(), "snapshot with the same name already exists");
             }
-            if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getUUID())) {
+            if (snapshotFormat.exists(blobContainer(), snapshotId.getUUID())) {
                 throw new InvalidSnapshotNameException(metadata.name(), snapshotId.getName(), "snapshot with the same name already exists");
             }
 
             // Write Global MetaData
-            globalMetaDataFormat.write(clusterMetaData, snapshotsBlobContainer, snapshotId.getUUID());
+            globalMetaDataFormat.write(clusterMetaData, blobContainer(), snapshotId.getUUID());
 
             // write the index metadata for each index in the snapshot
             for (IndexId index : indices) {
@@ -421,7 +491,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private void deleteSnapshotBlobIgnoringErrors(final SnapshotInfo snapshotInfo, final String blobId) {
         try {
-            snapshotFormat.delete(snapshotsBlobContainer, blobId);
+            snapshotFormat.delete(blobContainer(), blobId);
         } catch (IOException e) {
             if (snapshotInfo != null) {
                 logger.warn(() -> new ParameterizedMessage("[{}] Unable to delete snapshot file [{}]",
@@ -434,7 +504,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private void deleteGlobalMetaDataBlobIgnoringErrors(final SnapshotInfo snapshotInfo, final String blobId) {
         try {
-            globalMetaDataFormat.delete(snapshotsBlobContainer, blobId);
+            globalMetaDataFormat.delete(blobContainer(), blobId);
         } catch (IOException e) {
             if (snapshotInfo != null) {
                 logger.warn(() -> new ParameterizedMessage("[{}] Unable to delete global metadata file [{}]",
@@ -472,7 +542,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             startTime, failure, System.currentTimeMillis(), totalShards, shardFailures,
             includeGlobalState);
         try {
-            snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getUUID());
+            snapshotFormat.write(blobStoreSnapshot, blobContainer(), snapshotId.getUUID());
             final RepositoryData repositoryData = getRepositoryData();
             writeIndexGen(repositoryData.addSnapshot(snapshotId, blobStoreSnapshot.state(), indices), repositoryStateId);
         } catch (FileAlreadyExistsException ex) {
@@ -490,7 +560,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     @Override
     public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
         try {
-            return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
+            return snapshotFormat.read(blobContainer(), snapshotId.getUUID());
         } catch (NoSuchFileException ex) {
             throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
         } catch (IOException | NotXContentException ex) {
@@ -501,7 +571,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     @Override
     public MetaData getSnapshotGlobalMetaData(final SnapshotId snapshotId) {
         try {
-            return globalMetaDataFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
+            return globalMetaDataFormat.read(blobContainer(), snapshotId.getUUID());
         } catch (NoSuchFileException ex) {
             throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
         } catch (IOException ex) {
@@ -543,11 +613,21 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         return restoreRateLimitingTimeInNanos.count();
     }
 
+    protected void assertSnapshotOrGenericThread() {
+        assert Thread.currentThread().getName().contains(ThreadPool.Names.SNAPSHOT)
+            || Thread.currentThread().getName().contains(ThreadPool.Names.GENERIC) :
+            "Expected current thread [" + Thread.currentThread() + "] to be the snapshot or generic thread.";
+    }
+
     @Override
     public String startVerification() {
         try {
             if (isReadOnly()) {
-                // It's readonly - so there is not much we can do here to verify it
+                // TODO: add repository verification for read-only repositories
+
+                // It's readonly - so there is not much we can do here to verify it apart try to create blobStore()
+                // and check that is is accessible on the master
+                blobStore();
                 return null;
             } else {
                 String seed = UUIDs.randomBase64UUID();
@@ -584,7 +664,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             final String snapshotsIndexBlobName = INDEX_FILE_PREFIX + Long.toString(indexGen);
 
             RepositoryData repositoryData;
-            try (InputStream blob = snapshotsBlobContainer.readBlob(snapshotsIndexBlobName)) {
+            try (InputStream blob = blobContainer().readBlob(snapshotsIndexBlobName)) {
                 BytesStreamOutput out = new BytesStreamOutput();
                 Streams.copy(blob, out);
                 // EMPTY is safe here because RepositoryData#fromXContent calls namedObject
@@ -598,7 +678,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             }
 
             // now load the incompatible snapshot ids, if they exist
-            try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
+            try (InputStream blob = blobContainer().readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
                 BytesStreamOutput out = new BytesStreamOutput();
                 Streams.copy(blob, out);
                 try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
@@ -636,11 +716,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         return readOnly;
     }
 
-    // package private, only use for testing
-    BlobContainer blobContainer() {
-        return snapshotsBlobContainer;
-    }
-
     protected void writeIndexGen(final RepositoryData repositoryData, final long repositoryStateId) throws IOException {
         assert isReadOnly() == false; // can not write to a read only repository
         final long currentGen = latestIndexBlobId();
@@ -668,7 +743,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         // delete the N-2 index file if it exists, keep the previous one around as a backup
         if (isReadOnly() == false && newGen - 2 >= 0) {
             final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2);
-            snapshotsBlobContainer.deleteBlobIgnoringIfNotExists(oldSnapshotIndexFile);
+            blobContainer().deleteBlobIgnoringIfNotExists(oldSnapshotIndexFile);
         }
 
         // write the current generation to the index-latest file
@@ -736,7 +811,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     // package private for testing
     long readSnapshotIndexLatestBlob() throws IOException {
-        try (InputStream blob = snapshotsBlobContainer.readBlob(INDEX_LATEST_BLOB)) {
+        try (InputStream blob = blobContainer().readBlob(INDEX_LATEST_BLOB)) {
             BytesStreamOutput out = new BytesStreamOutput();
             Streams.copy(blob, out);
             return Numbers.bytesToLong(out.bytes().toBytesRef());
@@ -744,7 +819,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     private long listBlobsToGetLatestIndexId() throws IOException {
-        Map<String, BlobMetaData> blobs = snapshotsBlobContainer.listBlobsByPrefix(INDEX_FILE_PREFIX);
+        Map<String, BlobMetaData> blobs = blobContainer().listBlobsByPrefix(INDEX_FILE_PREFIX);
         long latest = RepositoryData.EMPTY_REPO_GEN;
         if (blobs.isEmpty()) {
             // no snapshot index blobs have been written yet
@@ -766,7 +841,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private void writeAtomic(final String blobName, final BytesReference bytesRef, boolean failIfAlreadyExists) throws IOException {
         try (InputStream stream = bytesRef.streamInput()) {
-            snapshotsBlobContainer.writeBlobAtomic(blobName, stream, bytesRef.length(), failIfAlreadyExists);
+            blobContainer().writeBlobAtomic(blobName, stream, bytesRef.length(), failIfAlreadyExists);
         }
     }
 
@@ -806,6 +881,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     @Override
     public void verify(String seed, DiscoveryNode localNode) {
+        assertSnapshotOrGenericThread();
         BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
         if (testBlobContainer.blobExists("master.dat")) {
             try  {
diff --git a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index 4d4ab60feef..643ff2bc93d 100644
--- a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -31,7 +31,6 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
-import java.io.IOException;
 import java.nio.file.Path;
 import java.util.function.Function;
 
@@ -61,8 +60,7 @@ public class FsRepository extends BlobStoreRepository {
     public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, Property.NodeScope);
     public static final Setting<Boolean> REPOSITORIES_COMPRESS_SETTING =
         Setting.boolSetting("repositories.fs.compress", false, Property.NodeScope);
-
-    private final FsBlobStore blobStore;
+    private final Environment environment;
 
     private ByteSizeValue chunkSize;
 
@@ -74,37 +72,45 @@ public class FsRepository extends BlobStoreRepository {
      * Constructs a shared file system repository.
      */
     public FsRepository(RepositoryMetaData metadata, Environment environment,
-                        NamedXContentRegistry namedXContentRegistry) throws IOException {
+                        NamedXContentRegistry namedXContentRegistry) {
         super(metadata, environment.settings(), namedXContentRegistry);
+        this.environment = environment;
         String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings());
         if (location.isEmpty()) {
-            logger.warn("the repository location is missing, it should point to a shared file system location that is available on all master and data nodes");
+            logger.warn("the repository location is missing, it should point to a shared file system location"
+                + " that is available on all master and data nodes");
             throw new RepositoryException(metadata.name(), "missing location");
         }
         Path locationFile = environment.resolveRepoFile(location);
         if (locationFile == null) {
             if (environment.repoFiles().length > 0) {
-                logger.warn("The specified location [{}] doesn't start with any repository paths specified by the path.repo setting: [{}] ", location, environment.repoFiles());
-                throw new RepositoryException(metadata.name(), "location [" + location + "] doesn't match any of the locations specified by path.repo");
+                logger.warn("The specified location [{}] doesn't start with any "
+                    + "repository paths specified by the path.repo setting: [{}] ", location, environment.repoFiles());
+                throw new RepositoryException(metadata.name(), "location [" + location
+                    + "] doesn't match any of the locations specified by path.repo");
             } else {
-                logger.warn("The specified location [{}] should start with a repository path specified by the path.repo setting, but the path.repo setting was not set on this node", location);
-                throw new RepositoryException(metadata.name(), "location [" + location + "] doesn't match any of the locations specified by path.repo because this setting is empty");
+                logger.warn("The specified location [{}] should start with a repository path specified by"
+                    + " the path.repo setting, but the path.repo setting was not set on this node", location);
+                throw new RepositoryException(metadata.name(), "location [" + location
+                    + "] doesn't match any of the locations specified by path.repo because this setting is empty");
             }
         }
 
-        blobStore = new FsBlobStore(settings, locationFile);
         if (CHUNK_SIZE_SETTING.exists(metadata.settings())) {
             this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings());
         } else {
             this.chunkSize = REPOSITORIES_CHUNK_SIZE_SETTING.get(settings);
         }
-        this.compress = COMPRESS_SETTING.exists(metadata.settings()) ? COMPRESS_SETTING.get(metadata.settings()) : REPOSITORIES_COMPRESS_SETTING.get(settings);
+        this.compress = COMPRESS_SETTING.exists(metadata.settings())
+            ? COMPRESS_SETTING.get(metadata.settings()) : REPOSITORIES_COMPRESS_SETTING.get(settings);
         this.basePath = BlobPath.cleanPath();
     }
 
     @Override
-    protected BlobStore blobStore() {
-        return blobStore;
+    protected BlobStore createBlobStore() throws Exception {
+        final String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings());
+        final Path locationFile = environment.resolveRepoFile(location);
+        return new FsBlobStore(settings, locationFile);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index cc971ed1b04..3a5302bcec2 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -406,7 +406,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
             Collections.emptySet());
         final ClusterService clusterService = mock(ClusterService.class);
         final RepositoriesService repositoriesService = new RepositoriesService(settings, clusterService,
-            transportService, null);
+            transportService, null, threadPool);
         final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService(settings, threadPool,
             transportService, null, clusterService);
         final ShardStateAction shardStateAction = mock(ShardStateAction.class);
diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java
index 7a1d3a89420..0eae9a14200 100644
--- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java
@@ -173,10 +173,17 @@ public class BlobStoreRepositoryRestoreTests extends IndexShardTestCase {
     }
 
     /** Create a {@link Repository} with a random name **/
-    private Repository createRepository() throws IOException {
+    private Repository createRepository() {
         Settings settings = Settings.builder().put("location", randomAlphaOfLength(10)).build();
         RepositoryMetaData repositoryMetaData = new RepositoryMetaData(randomAlphaOfLength(10), FsRepository.TYPE, settings);
-        return new FsRepository(repositoryMetaData, createEnvironment(), xContentRegistry());
+        final FsRepository repository = new FsRepository(repositoryMetaData, createEnvironment(), xContentRegistry()) {
+            @Override
+            protected void assertSnapshotOrGenericThread() {
+                // eliminate thread name check as we create repo manually
+            }
+        };
+        repository.start();
+        return repository;
     }
 
     /** Create a {@link Environment} with random path.home and path.repo **/
diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
index 7e4d5cc54a9..1abdb97f174 100644
--- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -24,10 +24,16 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -37,18 +43,42 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.repositories.RepositoryDataTests.generateRandomRepoData;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Tests for the {@link BlobStoreRepository} and its subclasses.
  */
 public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
 
+    static final String REPO_TYPE = "fsLike";
+
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Arrays.asList(FsLikeRepoPlugin.class);
+    }
+
+    // the reason for this plug-in is to drop any assertSnapshotOrGenericThread as mostly all access in this test goes from test threads
+    public static class FsLikeRepoPlugin extends org.elasticsearch.plugins.Plugin implements RepositoryPlugin {
+
+        @Override
+        public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
+            return Collections.singletonMap(REPO_TYPE,
+                (metadata) -> new FsRepository(metadata, env, namedXContentRegistry) {
+                    @Override
+                    protected void assertSnapshotOrGenericThread() {
+                        // eliminate thread name check as we access blobStore on test/main threads
+                    }
+                });
+        }
+    }
+
     public void testRetrieveSnapshots() throws Exception {
         final Client client = client();
         final Path location = ESIntegTestCase.randomRepoPath(node().settings());
@@ -57,7 +87,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         logger.info("-->  creating repository");
         PutRepositoryResponse putRepositoryResponse =
             client.admin().cluster().preparePutRepository(repositoryName)
-                                    .setType("fs")
+                                    .setType(REPO_TYPE)
                                     .setSettings(Settings.builder().put(node().settings()).put("location", location))
                                     .get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@@ -209,7 +239,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
 
         PutRepositoryResponse putRepositoryResponse =
             client.admin().cluster().preparePutRepository(repositoryName)
-                                    .setType("fs")
+                                    .setType(REPO_TYPE)
                                     .setSettings(Settings.builder().put(node().settings()).put("location", location))
                                     .get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
@@ -217,6 +247,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
         @SuppressWarnings("unchecked") final BlobStoreRepository repository =
             (BlobStoreRepository) repositoriesService.repository(repositoryName);
+        assertThat("getBlobContainer has to be lazy initialized", repository.getBlobContainer(), nullValue());
         return repository;
     }
 
diff --git a/server/src/test/java/org/elasticsearch/snapshots/FsBlobStoreRepositoryIT.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIT.java
similarity index 79%
rename from server/src/test/java/org/elasticsearch/snapshots/FsBlobStoreRepositoryIT.java
rename to server/src/test/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIT.java
index 792b1bdbddd..1ed42cb2474 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/FsBlobStoreRepositoryIT.java
+++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIT.java
@@ -16,22 +16,29 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.snapshots;
+package org.elasticsearch.repositories.fs;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class FsBlobStoreRepositoryIT extends ESBlobStoreRepositoryIntegTestCase {
     @Override
-    protected void createTestRepository(String name) {
+    protected void createTestRepository(String name, boolean verify) {
         assertAcked(client().admin().cluster().preparePutRepository(name)
+            .setVerify(verify)
             .setType("fs").setSettings(Settings.builder()
                 .put("location", randomRepoPath())
                 .put("compress", randomBoolean())
                 .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
+    }
 
+    @Override
+    protected void afterCreationCheck(Repository repository) {
+        assertThat(repository, instanceOf(FsRepository.class));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index c9ca1637b1a..d2954a4c128 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.snapshots;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionFuture;
@@ -93,6 +94,7 @@ import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.StoredScriptsIT;
 import org.elasticsearch.snapshots.mockstore.MockRepository;
 import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.nio.channels.SeekableByteChannel;
@@ -1262,7 +1264,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         RepositoriesService service = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
         Repository repository = service.repository("test-repo");
 
-        final Map<String, IndexId> indexIds = repository.getRepositoryData().getIndices();
+        final Map<String, IndexId> indexIds = getRepositoryData(repository).getIndices();
         final Path indicesPath = repo.resolve("indices");
 
         logger.info("--> delete index metadata and shard metadata");
@@ -1739,6 +1741,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> trying to create a repository with different name");
         assertAcked(client.admin().cluster().preparePutRepository("test-repo-2")
+                .setVerify(false) // do not do verification itself as snapshot threads could be fully blocked
                 .setType("fs").setSettings(Settings.builder().put("location", repositoryLocation.resolve("test"))));
 
         logger.info("--> unblocking blocked node");
@@ -2563,7 +2566,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> emulate an orphan snapshot");
         RepositoriesService repositoriesService = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
-        final RepositoryData repositoryData = repositoriesService.repository(repositoryName).getRepositoryData();
+        final RepositoryData repositoryData = getRepositoryData(repositoriesService.repository(repositoryName));
         final IndexId indexId = repositoryData.resolveIndexId(idxName);
 
         clusterService.submitStateUpdateTask("orphan snapshot test", new ClusterStateUpdateTask() {
@@ -2784,7 +2787,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         RepositoriesService service = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
         Repository repository = service.repository("test-repo");
 
-        final Map<String, IndexId> indexIds = repository.getRepositoryData().getIndices();
+        final RepositoryData repositoryData = getRepositoryData(repository);
+        final Map<String, IndexId> indexIds = repositoryData.getIndices();
         assertThat(indexIds.size(), equalTo(nbIndices));
 
         // Choose a random index from the snapshot
@@ -3445,6 +3449,19 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         }
     }
 
+    private RepositoryData getRepositoryData(Repository repository) throws InterruptedException {
+        ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName());
+        final SetOnce<RepositoryData> repositoryData = new SetOnce<>();
+        final CountDownLatch latch = new CountDownLatch(1);
+        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
+            repositoryData.set(repository.getRepositoryData());
+            latch.countDown();
+        });
+
+        latch.await();
+        return repositoryData.get();
+    }
+
     private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map<String, List<String>> indicesPerSnapshot) {
         for (SnapshotInfo snapshotInfo : response.getSnapshots()) {
             final List<String> expected = snapshotInfo.indices();
diff --git a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index d05a10905d8..75a86831bc5 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -92,8 +92,6 @@ public class MockRepository extends FsRepository {
 
     private final long waitAfterUnblock;
 
-    private final MockBlobStore mockBlobStore;
-
     private final String randomPrefix;
 
     private volatile boolean blockOnInitialization;
@@ -128,7 +126,6 @@ public class MockRepository extends FsRepository {
         waitAfterUnblock = metadata.settings().getAsLong("wait_after_unblock", 0L);
         allowAtomicOperations = metadata.settings().getAsBoolean("allow_atomic_operations", true);
         logger.info("starting mock repository with random prefix {}", randomPrefix);
-        mockBlobStore = new MockBlobStore(super.blobStore());
     }
 
     @Override
@@ -163,8 +160,8 @@ public class MockRepository extends FsRepository {
     }
 
     @Override
-    protected BlobStore blobStore() {
-        return mockBlobStore;
+    protected BlobStore createBlobStore() throws Exception {
+        return new MockBlobStore(super.createBlobStore());
     }
 
     public synchronized void unblock() {
@@ -195,7 +192,7 @@ public class MockRepository extends FsRepository {
     }
 
     private synchronized boolean blockExecution() {
-        logger.debug("Blocking execution");
+        logger.debug("[{}] Blocking execution", metadata.name());
         boolean wasBlocked = false;
         try {
             while (blockOnDataFiles || blockOnControlFiles || blockOnInitialization || blockOnWriteIndexFile ||
@@ -207,7 +204,7 @@ public class MockRepository extends FsRepository {
         } catch (InterruptedException ex) {
             Thread.currentThread().interrupt();
         }
-        logger.debug("Unblocking execution");
+        logger.debug("[{}] Unblocking execution", metadata.name());
         return wasBlocked;
     }
 
@@ -285,7 +282,7 @@ public class MockRepository extends FsRepository {
             }
 
             private void blockExecutionAndMaybeWait(final String blobName) {
-                logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path());
+                logger.info("[{}] blocking I/O operation for file [{}] at path [{}]", metadata.name(), blobName, path());
                 if (blockExecution() && waitAfterUnblock > 0) {
                     try {
                         // Delay operation after unblocking
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
index bf9c8193234..439728bac9e 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.repositories.blobstore;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder;
@@ -27,34 +28,61 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Basic integration tests for blob-based repository validation.
  */
 public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase {
 
-    protected abstract void createTestRepository(String name);
+    protected abstract void createTestRepository(String name, boolean verify);
+
+    protected void afterCreationCheck(Repository repository) {
+
+    }
+
+    protected void createAndCheckTestRepository(String name) {
+        final boolean verify = randomBoolean();
+        createTestRepository(name, verify);
+
+        final Iterable<RepositoriesService> repositoriesServices =
+            internalCluster().getDataOrMasterNodeInstances(RepositoriesService.class);
+
+        for (RepositoriesService repositoriesService : repositoriesServices) {
+            final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(name);
+
+            afterCreationCheck(repository);
+            assertThat("blob store has to be lazy initialized",
+                repository.getBlobStore(), verify ? is(notNullValue()) : is(nullValue()));
+        }
+
+    }
 
     public void testSnapshotAndRestore() throws Exception {
         final String repoName = randomAsciiName();
         logger.info("-->  creating repository {}", repoName);
-        createTestRepository(repoName);
+        createAndCheckTestRepository(repoName);
         int indexCount = randomIntBetween(1, 5);
         int[] docCounts = new int[indexCount];
         String[] indexNames = generateRandomNames(indexCount);
@@ -125,7 +153,7 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
     public void testMultipleSnapshotAndRollback() throws Exception {
         String repoName = randomAsciiName();
         logger.info("-->  creating repository {}", repoName);
-        createTestRepository(repoName);
+        createAndCheckTestRepository(repoName);
         int iterationCount = randomIntBetween(2, 5);
         int[] docCounts = new int[iterationCount];
         String indexName = randomAsciiName();
@@ -177,12 +205,12 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
         }
     }
 
-    public void testIndicesDeletedFromRepository() {
+    public void testIndicesDeletedFromRepository() throws Exception {
         Client client = client();
 
         logger.info("-->  creating repository");
         final String repoName = "test-repo";
-        createTestRepository(repoName);
+        createAndCheckTestRepository(repoName);
 
         createIndex("test-idx-1", "test-idx-2", "test-idx-3");
         ensureGreen();
@@ -219,12 +247,22 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
 
         logger.info("--> verify index folder deleted from blob container");
         RepositoriesService repositoriesSvc = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
+        ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName());
         @SuppressWarnings("unchecked") BlobStoreRepository repository = (BlobStoreRepository) repositoriesSvc.repository(repoName);
-        BlobContainer indicesBlobContainer = repository.blobStore().blobContainer(repository.basePath().add("indices"));
-        RepositoryData repositoryData = repository.getRepositoryData();
-        for (IndexId indexId : repositoryData.getIndices().values()) {
+
+        final SetOnce<BlobContainer> indicesBlobContainer = new SetOnce<>();
+        final SetOnce<RepositoryData> repositoryData = new SetOnce<>();
+        final CountDownLatch latch = new CountDownLatch(1);
+        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
+            indicesBlobContainer.set(repository.blobStore().blobContainer(repository.basePath().add("indices")));
+            repositoryData.set(repository.getRepositoryData());
+            latch.countDown();
+        });
+
+        latch.await();
+        for (IndexId indexId : repositoryData.get().getIndices().values()) {
             if (indexId.getName().equals("test-idx-3")) {
-                assertFalse(indicesBlobContainer.blobExists(indexId.getId())); // deleted index
+                assertFalse(indicesBlobContainer.get().blobExists(indexId.getId())); // deleted index
             }
         }
     }

From 1f72afa77314d27208e710398f65f701c865ea1f Mon Sep 17 00:00:00 2001
From: Michael Basnight <mbasnight@gmail.com>
Date: Fri, 13 Jul 2018 13:49:47 -0500
Subject: [PATCH 015/260] Watcher: Make settings reloadable (#31746)

This commit allows for rebuilding watcher secure secrets via the
reload_secure_settings API call. The commit also renames a method in the
Notification Service to make it a bit more readable.
---
 .../elasticsearch/xpack/watcher/Watcher.java  | 22 ++++++++++-
 .../notification/NotificationService.java     |  4 +-
 .../notification/email/EmailService.java      |  2 +-
 .../notification/hipchat/HipChatService.java  |  6 +--
 .../notification/jira/JiraService.java        |  2 +-
 .../pagerduty/PagerDutyService.java           |  2 +-
 .../notification/slack/SlackService.java      |  2 +-
 .../xpack/watcher/WatcherPluginTests.java     | 37 +++++++++++++++++++
 .../NotificationServiceTests.java             |  2 +-
 9 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index f7d51d328a7..78d1d37287f 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -38,6 +38,7 @@ import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
@@ -123,6 +124,7 @@ import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInputFactory;
 import org.elasticsearch.xpack.watcher.input.transform.TransformInput;
 import org.elasticsearch.xpack.watcher.input.transform.TransformInputFactory;
+import org.elasticsearch.xpack.watcher.notification.NotificationService;
 import org.elasticsearch.xpack.watcher.notification.email.Account;
 import org.elasticsearch.xpack.watcher.notification.email.EmailService;
 import org.elasticsearch.xpack.watcher.notification.email.HtmlSanitizer;
@@ -194,7 +196,7 @@ import java.util.function.UnaryOperator;
 
 import static java.util.Collections.emptyList;
 
-public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin {
+public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin, ReloadablePlugin {
 
     // This setting is only here for backward compatibility reasons as 6.x indices made use of it. It can be removed in 8.x.
     @Deprecated
@@ -221,6 +223,7 @@ public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin {
     protected final boolean transportClient;
     protected final boolean enabled;
     protected final Environment env;
+    protected List<NotificationService> reloadableServices = new ArrayList<>();
 
     public Watcher(final Settings settings) {
         this.settings = settings;
@@ -275,6 +278,12 @@ public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin {
         SlackService slackService = new SlackService(settings, httpClient, clusterService.getClusterSettings());
         PagerDutyService pagerDutyService = new PagerDutyService(settings, httpClient, clusterService.getClusterSettings());
 
+        reloadableServices.add(emailService);
+        reloadableServices.add(hipChatService);
+        reloadableServices.add(jiraService);
+        reloadableServices.add(slackService);
+        reloadableServices.add(pagerDutyService);
+
         TextTemplateEngine templateEngine = new TextTemplateEngine(settings, scriptService);
         Map<String, EmailAttachmentParser> emailAttachmentParsers = new HashMap<>();
         emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, httpTemplateParser,
@@ -613,4 +622,15 @@ public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin {
     public void close() throws IOException {
         IOUtils.closeWhileHandlingException(httpClient);
     }
+
+    /**
+     * Reloads all the reloadable services in watcher.
+     */
+    @Override
+    public void reload(Settings settings) {
+        if (enabled == false || transportClient) {
+            return;
+        }
+        reloadableServices.forEach(s -> s.reload(settings));
+    }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
index 88399d3cb93..027825ab778 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
@@ -31,7 +31,7 @@ public abstract class NotificationService<Account> extends AbstractComponent {
     public NotificationService(Settings settings, String type,
                                ClusterSettings clusterSettings, List<Setting<?>> pluginSettings) {
         this(settings, type);
-        clusterSettings.addSettingsUpdateConsumer(this::setAccountSetting, pluginSettings);
+        clusterSettings.addSettingsUpdateConsumer(this::reload, pluginSettings);
     }
 
     // Used for testing only
@@ -40,7 +40,7 @@ public abstract class NotificationService<Account> extends AbstractComponent {
         this.type = type;
     }
 
-    protected synchronized void setAccountSetting(Settings settings) {
+    public synchronized void reload(Settings settings) {
         Tuple<Map<String, Account>, Account> accounts = buildAccounts(settings, this::createAccount);
         this.accounts = Collections.unmodifiableMap(accounts.v1());
         this.defaultAccount = accounts.v2();
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
index 15859a5e044..e45ed55cee3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
@@ -127,7 +127,7 @@ public class EmailService extends NotificationService<Account> {
         clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_SEND_PARTIAL, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_WAIT_ON_QUIT, (s, o) -> {}, (s, o) -> {});
         // do an initial load
-        setAccountSetting(settings);
+        reload(settings);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatService.java
index ca970d5597b..2f21c2299a9 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatService.java
@@ -79,13 +79,13 @@ public class HipChatService extends NotificationService<HipChatAccount> {
         clusterSettings.addAffixUpdateConsumer(SETTING_PORT, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_MESSAGE_DEFAULTS, (s, o) -> {}, (s, o) -> {});
 
-        setAccountSetting(settings);
+        reload(settings);
     }
 
     @Override
-    protected synchronized void setAccountSetting(Settings settings) {
+    public synchronized void reload(Settings settings) {
         defaultServer = new HipChatServer(settings.getByPrefix("xpack.notification.hipchat."));
-        super.setAccountSetting(settings);
+        super.reload(settings);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraService.java
index 3ccff775051..49c05f36b24 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraService.java
@@ -75,7 +75,7 @@ public class JiraService extends NotificationService<JiraAccount> {
         clusterSettings.addAffixUpdateConsumer(SETTING_SECURE_PASSWORD, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_DEFAULTS, (s, o) -> {}, (s, o) -> {});
         // do an initial load
-        setAccountSetting(settings);
+        reload(settings);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyService.java
index 21c2f1fefb1..32a6dcb91aa 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyService.java
@@ -46,7 +46,7 @@ public class PagerDutyService extends NotificationService<PagerDutyAccount> {
         clusterSettings.addAffixUpdateConsumer(SETTING_SERVICE_API_KEY, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_SECURE_SERVICE_API_KEY, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_DEFAULTS, (s, o) -> {}, (s, o) -> {});
-        setAccountSetting(settings);
+        reload(settings);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SlackService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SlackService.java
index d648501a5f8..2a38e08d599 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SlackService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SlackService.java
@@ -46,7 +46,7 @@ public class SlackService extends NotificationService<SlackAccount> {
         clusterSettings.addAffixUpdateConsumer(SETTING_URL, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_URL_SECURE, (s, o) -> {}, (s, o) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_DEFAULTS, (s, o) -> {}, (s, o) -> {});
-        setAccountSetting(settings);
+        reload(settings);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
index abb981053e7..474f69c70ed 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
@@ -15,6 +15,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.watcher.notification.NotificationService;
 
 import java.util.List;
 
@@ -22,6 +23,10 @@ import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 
 public class WatcherPluginTests extends ESTestCase {
 
@@ -97,4 +102,36 @@ public class WatcherPluginTests extends ESTestCase {
                 .build();
         assertThat(Watcher.getWatcherThreadPoolSize(noDataNodeSettings), is(1));
     }
+
+    public void testReload() {
+        Settings settings = Settings.builder()
+            .put("xpack.watcher.enabled", true)
+            .put("path.home", createTempDir())
+            .build();
+        NotificationService mockService = mock(NotificationService.class);
+        Watcher watcher = new TestWatcher(settings, mockService);
+
+        watcher.reload(settings);
+        verify(mockService, times(1)).reload(settings);
+    }
+
+    public void testReloadDisabled() {
+        Settings settings = Settings.builder()
+            .put("xpack.watcher.enabled", false)
+            .put("path.home", createTempDir())
+            .build();
+        NotificationService mockService = mock(NotificationService.class);
+        Watcher watcher = new TestWatcher(settings, mockService);
+
+        watcher.reload(settings);
+        verifyNoMoreInteractions(mockService);
+    }
+
+    private class TestWatcher extends Watcher {
+
+        TestWatcher(Settings settings, NotificationService service) {
+            super(settings);
+            reloadableServices.add(service);
+        }
+    }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
index 829337e9acb..cb86913678a 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
@@ -82,7 +82,7 @@ public class NotificationServiceTests extends ESTestCase {
 
         TestNotificationService(Settings settings) {
             super(settings, "test");
-            setAccountSetting(settings);
+            reload(settings);
         }
 
         @Override

From feb07559aadaa3175d8d7d5401c1a55438ee6451 Mon Sep 17 00:00:00 2001
From: Paul Sanwald <paul.sanwald@elastic.co>
Date: Fri, 13 Jul 2018 14:59:11 -0400
Subject: [PATCH 016/260] fix typo

---
 .../aggregations/bucket/autodatehistogram-aggregation.asciidoc  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
index 28cb65ce6cc..3bd430d03d5 100644
--- a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
@@ -215,7 +215,7 @@ the specified time zone.
     "by_day": {
       "buckets": [
         {
-          "key_as_string": "2015-09-30T23:00:00.000-01:00",
+          "key_as_string": "2015-09-30T23:00:00.000-01:00", <1>
           "key": 1443657600000,
           "doc_count": 1
         },

From 42ca520377098edcbd851efa640259b6a81371a5 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 13 Jul 2018 13:07:26 -0700
Subject: [PATCH 017/260] Clean Up Snapshot Create Rest API (#31779)

Make SnapshotInfo and CreateSnapshotResponse parsers lenient for backwards compatibility.  Remove extraneous fields from CreateSnapshotRequest toXContent.
---
 .../elasticsearch/client/SnapshotClient.java  |  4 +-
 .../org/elasticsearch/client/SnapshotIT.java  |  4 +-
 .../SnapshotClientDocumentationIT.java        | 10 +++-
 .../snapshot/create_snapshot.asciidoc         | 11 ++++
 .../create/CreateSnapshotRequest.java         |  4 +-
 .../create/CreateSnapshotResponse.java        | 50 ++++++-------------
 .../action/support/IndicesOptions.java        | 28 +++++------
 .../elasticsearch/snapshots/SnapshotInfo.java | 25 ----------
 .../create/CreateSnapshotRequestTests.java    |  4 +-
 .../create/CreateSnapshotResponseTests.java   |  6 +--
 .../action/support/IndicesOptionsTests.java   |  3 --
 11 files changed, 56 insertions(+), 93 deletions(-)

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
index bc0bbe95488..f75f6cdef24 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
@@ -176,7 +176,7 @@ public final class SnapshotClient {
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
      * API on elastic.co</a>
      */
-    public CreateSnapshotResponse createSnapshot(CreateSnapshotRequest createSnapshotRequest, RequestOptions options)
+    public CreateSnapshotResponse create(CreateSnapshotRequest createSnapshotRequest, RequestOptions options)
         throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(createSnapshotRequest, RequestConverters::createSnapshot, options,
             CreateSnapshotResponse::fromXContent, emptySet());
@@ -188,7 +188,7 @@ public final class SnapshotClient {
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
      * API on elastic.co</a>
      */
-    public void createSnapshotAsync(CreateSnapshotRequest createSnapshotRequest, RequestOptions options,
+    public void createAsync(CreateSnapshotRequest createSnapshotRequest, RequestOptions options,
                                     ActionListener<CreateSnapshotResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(createSnapshotRequest, RequestConverters::createSnapshot, options,
             CreateSnapshotResponse::fromXContent, listener, emptySet());
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
index 45f9b5bbb0b..6d035f5db65 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
@@ -61,8 +61,8 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
     private CreateSnapshotResponse createTestSnapshot(CreateSnapshotRequest createSnapshotRequest) throws IOException {
         // assumes the repository already exists
 
-        return execute(createSnapshotRequest, highLevelClient().snapshot()::createSnapshot,
-            highLevelClient().snapshot()::createSnapshotAsync);
+        return execute(createSnapshotRequest, highLevelClient().snapshot()::create,
+            highLevelClient().snapshot()::createAsync);
     }
 
     public void testCreateRepository() throws IOException {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
index 403ebc7d774..68a8113af6d 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
@@ -425,7 +425,7 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
         // end::create-snapshot-request-waitForCompletion
 
         // tag::create-snapshot-execute
-        CreateSnapshotResponse response = client.snapshot().createSnapshot(request, RequestOptions.DEFAULT);
+        CreateSnapshotResponse response = client.snapshot().create(request, RequestOptions.DEFAULT);
         // end::create-snapshot-execute
 
         // tag::create-snapshot-response
@@ -433,6 +433,12 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
         // end::create-snapshot-response
 
         assertEquals(RestStatus.OK, status);
+
+        // tag::create-snapshot-response-snapshot-info
+        SnapshotInfo snapshotInfo = response.getSnapshotInfo(); // <1>
+        // end::create-snapshot-response-snapshot-info
+
+        assertNotNull(snapshotInfo);
     }
 
     public void testSnapshotCreateAsync() throws InterruptedException {
@@ -460,7 +466,7 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::create-snapshot-execute-async
-            client.snapshot().createSnapshotAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.snapshot().createAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::create-snapshot-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
diff --git a/docs/java-rest/high-level/snapshot/create_snapshot.asciidoc b/docs/java-rest/high-level/snapshot/create_snapshot.asciidoc
index dbd31380a9b..971a6ee4867 100644
--- a/docs/java-rest/high-level/snapshot/create_snapshot.asciidoc
+++ b/docs/java-rest/high-level/snapshot/create_snapshot.asciidoc
@@ -73,11 +73,22 @@ include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[create-snapshot-r
 [[java-rest-high-snapshot-create-snapshot-sync]]
 ==== Synchronous Execution
 
+Execute a `CreateSnapshotRequest` synchronously to receive a `CreateSnapshotResponse`.
+
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
 include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[create-snapshot-execute]
 --------------------------------------------------
 
+Retrieve the `SnapshotInfo` from a `CreateSnapshotResponse` when the snapshot is fully created.
+(The `waitForCompletion` parameter is `true`).
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[create-snapshot-response-snapshot-info]
+--------------------------------------------------
+<1> The `SnapshotInfo` object.
+
 [[java-rest-high-snapshot-create-snapshot-async]]
 ==== Asynchronous Execution
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 2ff01ab01ed..15fbac35bff 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -42,9 +42,9 @@ import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.common.Strings.EMPTY_ARRAY;
+import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 
 /**
@@ -433,8 +433,6 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
         if (indicesOptions != null) {
             indicesOptions.toXContent(builder, params);
         }
-        builder.field("wait_for_completion", waitForCompletion);
-        builder.field("master_node_timeout", masterNodeTimeout.toString());
         builder.endObject();
         return builder;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
index a2dc02c5c82..d0a82e36a97 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
@@ -21,14 +21,16 @@ package org.elasticsearch.action.admin.cluster.snapshots.create;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotInfo.SnapshotInfoBuilder;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -38,6 +40,14 @@ import java.util.Objects;
  */
 public class CreateSnapshotResponse extends ActionResponse implements ToXContentObject {
 
+    private static final ObjectParser<CreateSnapshotResponse, Void> PARSER =
+        new ObjectParser<>(CreateSnapshotResponse.class.getName(), true, CreateSnapshotResponse::new);
+
+    static {
+        PARSER.declareObject(CreateSnapshotResponse::setSnapshotInfoFromBuilder,
+            SnapshotInfo.SNAPSHOT_INFO_PARSER, new ParseField("snapshot"));
+    }
+
     @Nullable
     private SnapshotInfo snapshotInfo;
 
@@ -48,8 +58,8 @@ public class CreateSnapshotResponse extends ActionResponse implements ToXContent
     CreateSnapshotResponse() {
     }
 
-    void setSnapshotInfo(SnapshotInfo snapshotInfo) {
-        this.snapshotInfo = snapshotInfo;
+    private void setSnapshotInfoFromBuilder(SnapshotInfoBuilder snapshotInfoBuilder) {
+        this.snapshotInfo = snapshotInfoBuilder.build();
     }
 
     /**
@@ -101,38 +111,8 @@ public class CreateSnapshotResponse extends ActionResponse implements ToXContent
         return builder;
     }
 
-    public static CreateSnapshotResponse fromXContent(XContentParser parser) throws IOException {
-        CreateSnapshotResponse createSnapshotResponse = new CreateSnapshotResponse();
-
-        parser.nextToken(); // move to '{'
-
-        if (parser.currentToken() != Token.START_OBJECT) {
-            throw new IllegalArgumentException("unexpected token [" + parser.currentToken() + "], expected ['{']");
-        }
-
-        parser.nextToken(); // move to 'snapshot' || 'accepted'
-
-        if ("snapshot".equals(parser.currentName())) {
-            createSnapshotResponse.snapshotInfo = SnapshotInfo.fromXContent(parser);
-        } else if ("accepted".equals(parser.currentName())) {
-            parser.nextToken(); // move to 'accepted' field value
-
-            if (parser.booleanValue()) {
-                // ensure accepted is a boolean value
-            }
-
-            parser.nextToken(); // move past 'true'/'false'
-        } else {
-            throw new IllegalArgumentException("unexpected token [" + parser.currentToken() + "] expected ['snapshot', 'accepted']");
-        }
-
-        if (parser.currentToken() != Token.END_OBJECT) {
-            throw new IllegalArgumentException("unexpected token [" + parser.currentToken() + "], expected ['}']");
-        }
-
-        parser.nextToken(); // move past '}'
-
-        return createSnapshotResponse;
+    public static CreateSnapshotResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index 19572a6c212..f2cf0b5444d 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.support;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestRequest;
@@ -316,21 +317,6 @@ public class IndicesOptions implements ToXContentFragment {
                 defaultSettings);
     }
 
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray("expand_wildcards");
-        for (WildcardStates expandWildcard : expandWildcards) {
-            builder.value(expandWildcard.toString().toLowerCase(Locale.ROOT));
-        }
-        builder.endArray();
-        builder.field("ignore_unavailable", ignoreUnavailable());
-        builder.field("allow_no_indices", allowNoIndices());
-        builder.field("forbid_aliases_to_multiple_indices", allowAliasesToMultipleIndices() == false);
-        builder.field("forbid_closed_indices", forbidClosedIndices());
-        builder.field("ignore_aliases", ignoreAliases());
-        return builder;
-    }
-
     /**
      * Returns true if the name represents a valid name for one of the indices option
      * false otherwise
@@ -360,6 +346,18 @@ public class IndicesOptions implements ToXContentFragment {
         );
     }
 
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startArray("expand_wildcards");
+        for (WildcardStates expandWildcard : expandWildcards) {
+            builder.value(expandWildcard.toString().toLowerCase(Locale.ROOT));
+        }
+        builder.endArray();
+        builder.field("ignore_unavailable", ignoreUnavailable());
+        builder.field("allow_no_indices", allowNoIndices());
+        return builder;
+    }
+
     /**
      * @return indices options that requires every specified index to exist, expands wildcards only to open indices and
      *         allows that no indices are resolved from wildcard expressions (not returning an error).
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index cf2f66a750c..67ddabc37fa 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -140,22 +140,6 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
             this.shardFailures = shardFailures;
         }
 
-        private void ignoreVersion(String version) {
-            // ignore extra field
-        }
-
-        private void ignoreStartTime(String startTime) {
-            // ignore extra field
-        }
-
-        private void ignoreEndTime(String endTime) {
-            // ignore extra field
-        }
-
-        private void ignoreDurationInMillis(long durationInMillis) {
-            // ignore extra field
-        }
-
         public SnapshotInfo build() {
             SnapshotId snapshotId = new SnapshotId(snapshotName, snapshotUUID);
 
@@ -197,10 +181,6 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         int getSuccessfulShards() {
             return successfulShards;
         }
-
-        private void ignoreFailedShards(int failedShards) {
-            // ignore extra field
-        }
     }
 
     public static final ObjectParser<SnapshotInfoBuilder, Void> SNAPSHOT_INFO_PARSER =
@@ -222,14 +202,9 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         SNAPSHOT_INFO_PARSER.declareInt(SnapshotInfoBuilder::setVersion, new ParseField(VERSION_ID));
         SNAPSHOT_INFO_PARSER.declareObjectArray(SnapshotInfoBuilder::setShardFailures, SnapshotShardFailure.SNAPSHOT_SHARD_FAILURE_PARSER,
             new ParseField(FAILURES));
-        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::ignoreVersion, new ParseField(VERSION));
-        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::ignoreStartTime, new ParseField(START_TIME));
-        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::ignoreEndTime, new ParseField(END_TIME));
-        SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::ignoreDurationInMillis, new ParseField(DURATION_IN_MILLIS));
 
         SHARD_STATS_PARSER.declareInt(ShardStatsBuilder::setTotalShards, new ParseField(TOTAL));
         SHARD_STATS_PARSER.declareInt(ShardStatsBuilder::setSuccessfulShards, new ParseField(SUCCESSFUL));
-        SHARD_STATS_PARSER.declareInt(ShardStatsBuilder::ignoreFailedShards, new ParseField(FAILED));
     }
 
     private final SnapshotId snapshotId;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
index 1bde8ab572b..0b598be6849 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
@@ -102,8 +102,8 @@ public class CreateSnapshotRequestTests extends ESTestCase {
                 NamedXContentRegistry.EMPTY, null, BytesReference.bytes(builder).streamInput());
         Map<String, Object> map = parser.mapOrdered();
         CreateSnapshotRequest processed = new CreateSnapshotRequest((String)map.get("repository"), (String)map.get("snapshot"));
-        processed.waitForCompletion((boolean)map.getOrDefault("wait_for_completion", false));
-        processed.masterNodeTimeout((String)map.getOrDefault("master_node_timeout", "30s"));
+        processed.waitForCompletion(original.waitForCompletion());
+        processed.masterNodeTimeout(original.masterNodeTimeout());
         processed.source(map);
 
         assertEquals(original, processed);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
index bbfc9755bf2..bbb11fc6fee 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
@@ -40,7 +40,7 @@ public class CreateSnapshotResponseTests extends AbstractXContentTestCase<Create
 
     @Override
     protected boolean supportsUnknownFields() {
-        return false;
+        return true;
     }
 
     @Override
@@ -63,9 +63,7 @@ public class CreateSnapshotResponseTests extends AbstractXContentTestCase<Create
 
         boolean globalState = randomBoolean();
 
-        CreateSnapshotResponse response = new CreateSnapshotResponse();
-        response.setSnapshotInfo(
+        return new CreateSnapshotResponse(
             new SnapshotInfo(snapshotId, indices, startTime, reason, endTime, totalShards, shardFailures, globalState));
-        return response;
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
index 8e94764cc7a..f564066b4e3 100644
--- a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
@@ -320,8 +320,5 @@ public class IndicesOptionsTests extends ESTestCase {
         }
         assertEquals(map.get("ignore_unavailable"), options.contains(Option.IGNORE_UNAVAILABLE));
         assertEquals(map.get("allow_no_indices"), options.contains(Option.ALLOW_NO_INDICES));
-        assertEquals(map.get("forbid_aliases_to_multiple_indices"), options.contains(Option.FORBID_ALIASES_TO_MULTIPLE_INDICES));
-        assertEquals(map.get("forbid_closed_indices"), options.contains(Option.FORBID_CLOSED_INDICES));
-        assertEquals(map.get("ignore_aliases"), options.contains(Option.IGNORE_ALIASES));
     }
 }

From 867c49600c69e7e7baf6559bc1c2b736d6405c3f Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Fri, 13 Jul 2018 16:10:39 -0400
Subject: [PATCH 018/260] [Rollup] Histo group config should support
 scaled_floats (#32048)

Metric config already whitelist scaled_floats, but it wasn't added to
the histo group config.  This centralizes the mapping types map
so that both metrics and histo (and any future configs) use the same
map.

Fixes #32035
---
 .../xpack/core/rollup/RollupField.java              | 13 +++++++++++++
 .../xpack/core/rollup/job/HistoGroupConfig.java     |  8 +-------
 .../xpack/core/rollup/job/MetricConfig.java         | 13 +------------
 .../job/HistoGroupConfigSerializingTests.java       |  4 +++-
 4 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java
index 1e2e011276d..134ce6c87b3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.rollup;
 
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
@@ -15,6 +16,8 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuil
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class RollupField {
     // Fields that are used both in core Rollup actions and Rollup plugin
@@ -34,6 +37,16 @@ public class RollupField {
     public static final List<String> SUPPORTED_METRICS = Arrays.asList(MaxAggregationBuilder.NAME, MinAggregationBuilder.NAME,
             SumAggregationBuilder.NAME, AvgAggregationBuilder.NAME, ValueCountAggregationBuilder.NAME);
 
+    // these mapper types are used by the configs (metric, histo, etc) to validate field mappings
+    public static final List<String> NUMERIC_FIELD_MAPPER_TYPES;
+    static {
+        List<String> types = Stream.of(NumberFieldMapper.NumberType.values())
+            .map(NumberFieldMapper.NumberType::typeName)
+            .collect(Collectors.toList());
+        types.add("scaled_float"); // have to add manually since scaled_float is in a module
+        NUMERIC_FIELD_MAPPER_TYPES = types;
+    }
+
     /**
      * Format to the appropriate Rollup field name convention
      *
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java
index 2b1511077d9..87de9e16534 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfig.java
@@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
 import org.elasticsearch.search.aggregations.bucket.composite.HistogramValuesSourceBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
@@ -30,7 +29,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * The configuration object for the histograms in the rollup config
@@ -51,10 +49,6 @@ public class HistoGroupConfig implements Writeable, ToXContentFragment {
 
     private static final ParseField INTERVAL = new ParseField("interval");
     private static final ParseField FIELDS = new ParseField("fields");
-    private static final List<String> MAPPER_TYPES = Stream.of(NumberFieldMapper.NumberType.values())
-            .map(NumberFieldMapper.NumberType::typeName)
-            .collect(Collectors.toList());
-
 
     private final long interval;
     private final String[] fields;
@@ -126,7 +120,7 @@ public class HistoGroupConfig implements Writeable, ToXContentFragment {
             Map<String, FieldCapabilities> fieldCaps = fieldCapsResponse.get(field);
             if (fieldCaps != null && fieldCaps.isEmpty() == false) {
                 fieldCaps.forEach((key, value) -> {
-                    if (MAPPER_TYPES.contains(key)) {
+                    if (RollupField.NUMERIC_FIELD_MAPPER_TYPES.contains(key)) {
                         if (value.isAggregatable() == false) {
                             validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " +
                                     "but is not.");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java
index 67b83646c42..006d8c35c32 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java
@@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
@@ -32,7 +31,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * The configuration object for the metrics portion of a rollup job config
@@ -66,15 +64,6 @@ public class MetricConfig implements Writeable, ToXContentFragment {
     private static final ParseField AVG = new ParseField("avg");
     private static final ParseField VALUE_COUNT = new ParseField("value_count");
 
-    private static final List<String> MAPPER_TYPES;
-    static {
-        List<String> types = Stream.of(NumberFieldMapper.NumberType.values())
-                .map(NumberFieldMapper.NumberType::typeName)
-                .collect(Collectors.toList());
-        types.add("scaled_float"); // have to add manually since scaled_float is in a module
-        MAPPER_TYPES = types;
-    }
-
     public static final ObjectParser<MetricConfig.Builder, Void> PARSER = new ObjectParser<>(NAME, MetricConfig.Builder::new);
 
     static {
@@ -153,7 +142,7 @@ public class MetricConfig implements Writeable, ToXContentFragment {
         Map<String, FieldCapabilities> fieldCaps = fieldCapsResponse.get(field);
         if (fieldCaps != null && fieldCaps.isEmpty() == false) {
             fieldCaps.forEach((key, value) -> {
-                if (MAPPER_TYPES.contains(key)) {
+                if (RollupField.NUMERIC_FIELD_MAPPER_TYPES.contains(key)) {
                     if (value.isAggregatable() == false) {
                         validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " +
                                 "but is not.");
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfigSerializingTests.java
index 18a64bc2adf..92e7d8b9643 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistoGroupConfigSerializingTests.java
@@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
+import org.elasticsearch.xpack.core.rollup.RollupField;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -111,7 +112,8 @@ public class HistoGroupConfigSerializingTests extends AbstractSerializingTestCas
         // Have to mock fieldcaps because the ctor's aren't public...
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("long", fieldCaps));
+        String mappingType = randomFrom(RollupField.NUMERIC_FIELD_MAPPER_TYPES);
+        responseMap.put("my_field", Collections.singletonMap(mappingType, fieldCaps));
 
         HistoGroupConfig config = new HistoGroupConfig.Builder()
                 .setFields(Collections.singletonList("my_field"))

From 466235288e8fc74d265fbf6f98d044572a60a460 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Fri, 13 Jul 2018 23:08:18 +0200
Subject: [PATCH 019/260] Mute failing tests

Relates to #32055
---
 .../test/search.inner_hits/10_basic.yml         |  9 +++++----
 .../test/search/110_field_collapsing.yml        | 17 ++++++++++++++---
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml
index 884a50507c7..8f162ae2eb2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml
@@ -13,8 +13,9 @@ setup:
 ---
 "Nested inner hits":
     - skip:
-        version: " - 6.1.99"
-        reason: "<= 6.1 nodes don't always include index or id in nested inner hits"
+        version: "all"
+        reason: "https://github.com/elastic/elasticsearch/issues/32055"
+
     - do:
         index:
           index: test
@@ -45,8 +46,8 @@ setup:
 "Nested doc version and seqIDs":
 
     - skip:
-        version: " - 6.3.99"
-        reason:  "object notation for docvalue_fields was introduced in 6.4"
+        version: "all"
+        reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
     - do:
         index:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
index 2dfd868d66b..39597b1fbbe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
@@ -107,6 +107,9 @@ setup:
 
 ---
 "field collapsing and inner_hits":
+  - skip:
+      version: "all"
+      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:
@@ -146,6 +149,9 @@ setup:
 
 ---
 "field collapsing, inner_hits and maxConcurrentGroupRequests":
+  - skip:
+      version: "all"
+      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:
@@ -226,6 +232,9 @@ setup:
 
 ---
 "no hits and inner_hits":
+  - skip:
+      version: "all"
+      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:
@@ -240,6 +249,9 @@ setup:
 
 ---
 "field collapsing and multiple inner_hits":
+  - skip:
+      version: "all"
+      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:
@@ -292,10 +304,9 @@ setup:
 
 ---
 "field collapsing, inner_hits and version":
-
   - skip:
-      version: " - 6.1.0"
-      reason:  "bug fixed in 6.1.1"
+      version: "all"
+      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:

From 3679d00a7465c9d8602e991be76390bd905fddd7 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 13 Jul 2018 23:26:10 +0200
Subject: [PATCH 020/260] Replace Ingest ScriptContext with Custom Interface
 (#32003)

* Replace Ingest ScriptContext with Custom Interface
* Make org.elasticsearch.ingest.common.ScriptProcessorTests#testScripting more precise
* Don't mock script factory in ScriptProcessorTests
* Adjust mock script plugin in IT for new API
---
 .../ingest/common/ScriptProcessor.java        | 10 ++--
 .../ingest/common/IngestRestartIT.java        |  4 +-
 .../ingest/common/ScriptProcessorTests.java   | 38 ++++++++------
 .../script/ExecutableScript.java              |  1 -
 .../elasticsearch/script/IngestScript.java    | 52 +++++++++++++++++++
 .../elasticsearch/script/ScriptModule.java    |  2 +-
 .../script/ScriptServiceTests.java            |  8 +--
 .../script/MockScriptEngine.java              |  8 +++
 8 files changed, 91 insertions(+), 32 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/script/IngestScript.java

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index ddb284b9c89..74c68fd5c26 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -31,7 +31,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.IngestScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.ScriptService;
@@ -73,10 +73,8 @@ public final class ScriptProcessor extends AbstractProcessor {
      */
     @Override
     public void execute(IngestDocument document) {
-        ExecutableScript.Factory factory = scriptService.compile(script, ExecutableScript.INGEST_CONTEXT);
-        ExecutableScript executableScript = factory.newInstance(script.getParams());
-        executableScript.setNextVar("ctx",  document.getSourceAndMetadata());
-        executableScript.run();
+        IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
+        factory.newInstance(script.getParams()).execute(document.getSourceAndMetadata());
     }
 
     @Override
@@ -108,7 +106,7 @@ public final class ScriptProcessor extends AbstractProcessor {
 
                 // verify script is able to be compiled before successfully creating processor.
                 try {
-                    scriptService.compile(script, ExecutableScript.INGEST_CONTEXT);
+                    scriptService.compile(script, IngestScript.CONTEXT);
                 } catch (ScriptException e) {
                     throw newConfigurationException(TYPE, processorTag, null, e);
                 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java
index 69236144007..8c3976d2b17 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java
@@ -58,9 +58,7 @@ public class IngestRestartIT extends ESIntegTestCase {
     public static class CustomScriptPlugin extends MockScriptPlugin {
         @Override
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
-            return Collections.singletonMap("my_script", script -> {
-                @SuppressWarnings("unchecked")
-                Map<String, Object> ctx = (Map<String, Object>) script.get("ctx");
+            return Collections.singletonMap("my_script", ctx -> {
                 ctx.put("z", 0);
                 return null;
             });
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
index 1004a41bcc5..72bc337e9c9 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
@@ -19,22 +19,22 @@
 
 package org.elasticsearch.ingest.common;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.core.Is.is;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class ScriptProcessorTests extends ESTestCase {
 
@@ -42,24 +42,28 @@ public class ScriptProcessorTests extends ESTestCase {
         int randomBytesIn = randomInt();
         int randomBytesOut = randomInt();
         int randomBytesTotal = randomBytesIn + randomBytesOut;
-
-        ScriptService scriptService = mock(ScriptService.class);
-        Script script = mockScript("_script");
-        ExecutableScript.Factory factory = mock(ExecutableScript.Factory.class);
-        ExecutableScript executableScript = mock(ExecutableScript.class);
-        when(scriptService.compile(script, ExecutableScript.INGEST_CONTEXT)).thenReturn(factory);
-        when(factory.newInstance(any())).thenReturn(executableScript);
+        String scriptName = "script";
+        ScriptService scriptService = new ScriptService(Settings.builder().build(),
+            Collections.singletonMap(
+                Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
+                    Script.DEFAULT_SCRIPT_LANG,
+                    Collections.singletonMap(
+                        scriptName, ctx -> {
+                            ctx.put("bytes_total", randomBytesTotal);
+                            return null;
+                        }
+                    )
+                )
+            ),
+            new HashMap<>(ScriptModule.CORE_CONTEXTS)
+        );
+        Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
 
         Map<String, Object> document = new HashMap<>();
         document.put("bytes_in", randomInt());
         document.put("bytes_out", randomInt());
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
-        doAnswer(invocationOnMock ->  {
-            ingestDocument.setFieldValue("bytes_total", randomBytesTotal);
-            return null;
-        }).when(executableScript).run();
-
         ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, scriptService);
 
         processor.execute(ingestDocument);
diff --git a/server/src/main/java/org/elasticsearch/script/ExecutableScript.java b/server/src/main/java/org/elasticsearch/script/ExecutableScript.java
index e87b7cdf389..2f7a01c3798 100644
--- a/server/src/main/java/org/elasticsearch/script/ExecutableScript.java
+++ b/server/src/main/java/org/elasticsearch/script/ExecutableScript.java
@@ -50,5 +50,4 @@ public interface ExecutableScript {
     // TODO: remove these once each has its own script interface
     ScriptContext<Factory> AGGS_CONTEXT = new ScriptContext<>("aggs_executable", Factory.class);
     ScriptContext<Factory> UPDATE_CONTEXT = new ScriptContext<>("update", Factory.class);
-    ScriptContext<Factory> INGEST_CONTEXT = new ScriptContext<>("ingest", Factory.class);
 }
diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java
new file mode 100644
index 00000000000..f357394ed31
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java
@@ -0,0 +1,52 @@
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import java.util.Map;
+
+/**
+ * A script used by the Ingest Script Processor.
+ */
+public abstract class IngestScript {
+
+    public static final String[] PARAMETERS = { "ctx" };
+
+    /** The context used to compile {@link IngestScript} factories. */
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("ingest", Factory.class);
+
+    /** The generic runtime parameters for the script. */
+    private final Map<String, Object> params;
+
+    public IngestScript(Map<String, Object> params) {
+        this.params = params;
+    }
+
+    /** Return the parameters for this script. */
+    public Map<String, Object> getParams() {
+        return params;
+    }
+
+    public abstract void execute(Map<String, Object> ctx);
+
+    public interface Factory {
+        IngestScript newInstance(Map<String, Object> params);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java
index 042953117c5..bf4bd9c57ce 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -51,7 +51,7 @@ public class ScriptModule {
             ExecutableScript.CONTEXT,
             ExecutableScript.AGGS_CONTEXT,
             ExecutableScript.UPDATE_CONTEXT,
-            ExecutableScript.INGEST_CONTEXT,
+            IngestScript.CONTEXT,
             FilterScript.CONTEXT,
             SimilarityScript.CONTEXT,
             SimilarityWeightScript.CONTEXT,
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index b35fcbcc03c..585f8601651 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -168,7 +168,7 @@ public class ScriptServiceTests extends ESTestCase {
         assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.CONTEXT);
         assertCompileAccepted("painless", "script", ScriptType.INLINE, SearchScript.AGGS_CONTEXT);
         assertCompileAccepted("painless", "script", ScriptType.INLINE, ExecutableScript.UPDATE_CONTEXT);
-        assertCompileAccepted("painless", "script", ScriptType.INLINE, ExecutableScript.INGEST_CONTEXT);
+        assertCompileAccepted("painless", "script", ScriptType.INLINE, IngestScript.CONTEXT);
     }
 
     public void testAllowSomeScriptTypeSettings() throws IOException {
@@ -209,13 +209,13 @@ public class ScriptServiceTests extends ESTestCase {
     }
 
     public void testCompileNonRegisteredContext() throws IOException {
-        contexts.remove(ExecutableScript.INGEST_CONTEXT.name);
+        contexts.remove(IngestScript.CONTEXT.name);
         buildScriptService(Settings.EMPTY);
 
         String type = scriptEngine.getType();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
-            scriptService.compile(new Script(ScriptType.INLINE, type, "test", Collections.emptyMap()), ExecutableScript.INGEST_CONTEXT));
-        assertThat(e.getMessage(), containsString("script context [" + ExecutableScript.INGEST_CONTEXT.name + "] not supported"));
+            scriptService.compile(new Script(ScriptType.INLINE, type, "test", Collections.emptyMap()), IngestScript.CONTEXT));
+        assertThat(e.getMessage(), containsString("script context [" + IngestScript.CONTEXT.name + "] not supported"));
     }
 
     public void testCompileCountedInCompilationStats() throws IOException {
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index e608bd13d25..8e40e4bcf14 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -88,6 +88,14 @@ public class MockScriptEngine implements ScriptEngine {
         } else if (context.instanceClazz.equals(ExecutableScript.class)) {
             ExecutableScript.Factory factory = mockCompiled::createExecutableScript;
             return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(IngestScript.class)) {
+            IngestScript.Factory factory = parameters -> new IngestScript(parameters) {
+                @Override
+                public void execute(Map<String, Object> ctx) {
+                    script.apply(ctx);
+                }
+            };
+            return context.factoryClazz.cast(factory);
         } else if (context.instanceClazz.equals(TemplateScript.class)) {
             TemplateScript.Factory factory = vars -> {
                 // TODO: need a better way to implement all these new contexts

From 305bfea9c3c174d35136fddbf2bcd8544b4ea3bd Mon Sep 17 00:00:00 2001
From: Tim Brooks <tim@uncontended.net>
Date: Fri, 13 Jul 2018 16:41:02 -0600
Subject: [PATCH 021/260] Add nio http transport to security plugin (#32018)

This is related to #27260. It adds the SecurityNioHttpServerTransport
to the security plugin. It randomly uses the nio http transport in
security integration tests.
---
 .../transport/netty4/Netty4TcpChannel.java    |   2 +-
 .../http/nio/HttpReadWriteHandler.java        |   4 +-
 .../http/nio/NioHttpChannel.java              |   2 +-
 .../http/nio/NioHttpServerChannel.java        |   3 +-
 .../http/nio/NioHttpServerTransport.java      |  27 ++-
 .../xpack/security/Security.java              |  21 +-
 .../security/rest/SecurityRestFilter.java     |  10 +-
 .../security/transport/SSLEngineUtils.java    |  93 ++++++++
 .../SecurityHttpExceptionHandler.java         |  64 ++++++
 .../transport/SecurityHttpSettings.java       |  22 ++
 .../transport/ServerTransportFilter.java      |  46 +---
 .../SecurityNetty4HttpServerTransport.java    |  47 +---
 .../security/transport/nio/NioIPFilter.java   |  32 +++
 .../transport/nio/SSLChannelContext.java      |   5 +
 .../security/transport/nio/SSLDriver.java     |   4 +
 .../nio/SecurityNioHttpServerTransport.java   | 132 +++++++++++
 .../transport/nio/SecurityNioTransport.java   |  19 +-
 .../test/SecurityIntegTestCase.java           |   1 +
 .../test/SecuritySettingsSource.java          |   1 +
 .../transport/SecurityHttpSettingsTests.java  |  44 ++++
 ...ecurityNetty4HttpServerTransportTests.java |  29 ---
 .../transport/nio/NioIPFilterTests.java       |  91 ++++++++
 .../SecurityNioHttpServerTransportTests.java  | 207 ++++++++++++++++++
 23 files changed, 750 insertions(+), 156 deletions(-)
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SSLEngineUtils.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpExceptionHandler.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpSettings.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilter.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityHttpSettingsTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilterTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransportTests.java

diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java
index 78a14255000..51821c73329 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java
@@ -112,7 +112,7 @@ public class Netty4TcpChannel implements TcpChannel {
         }
     }
 
-    public Channel getLowLevelChannel() {
+    public Channel getNettyChannel() {
         return channel;
     }
 
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java
index ad81719ebcb..3dcd59cf8e2 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java
@@ -51,8 +51,8 @@ public class HttpReadWriteHandler implements ReadWriteHandler {
     private final NioHttpChannel nioHttpChannel;
     private final NioHttpServerTransport transport;
 
-    HttpReadWriteHandler(NioHttpChannel nioHttpChannel, NioHttpServerTransport transport, HttpHandlingSettings settings,
-                         NioCorsConfig corsConfig) {
+    public HttpReadWriteHandler(NioHttpChannel nioHttpChannel, NioHttpServerTransport transport, HttpHandlingSettings settings,
+                                NioCorsConfig corsConfig) {
         this.nioHttpChannel = nioHttpChannel;
         this.transport = transport;
 
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpChannel.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpChannel.java
index 0a797a5687e..1a4c5f14c91 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpChannel.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpChannel.java
@@ -28,7 +28,7 @@ import java.nio.channels.SocketChannel;
 
 public class NioHttpChannel extends NioSocketChannel implements HttpChannel {
 
-    NioHttpChannel(SocketChannel socketChannel) {
+    public NioHttpChannel(SocketChannel socketChannel) {
         super(socketChannel);
     }
 
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerChannel.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerChannel.java
index 2674d38dc49..d72376da5c0 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerChannel.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerChannel.java
@@ -23,12 +23,11 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.http.HttpServerChannel;
 import org.elasticsearch.nio.NioServerSocketChannel;
 
-import java.io.IOException;
 import java.nio.channels.ServerSocketChannel;
 
 public class NioHttpServerChannel extends NioServerSocketChannel implements HttpServerChannel {
 
-    NioHttpServerChannel(ServerSocketChannel serverSocketChannel) throws IOException {
+    public NioHttpServerChannel(ServerSocketChannel serverSocketChannel) {
         super(serverSocketChannel);
     }
 
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java
index b80778e9642..9c672c1caf1 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java
@@ -35,7 +35,6 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.http.AbstractHttpServerTransport;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.http.HttpServerChannel;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.nio.cors.NioCorsConfig;
 import org.elasticsearch.http.nio.cors.NioCorsConfigBuilder;
 import org.elasticsearch.nio.BytesChannelContext;
@@ -87,21 +86,21 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
             (s) -> Integer.toString(EsExecutors.numberOfProcessors(s) * 2),
             (s) -> Setting.parseInt(s, 1, "http.nio.worker_count"), Setting.Property.NodeScope);
 
-    private final PageCacheRecycler pageCacheRecycler;
+    protected final PageCacheRecycler pageCacheRecycler;
+    protected final NioCorsConfig corsConfig;
 
-    private final boolean tcpNoDelay;
-    private final boolean tcpKeepAlive;
-    private final boolean reuseAddress;
-    private final int tcpSendBufferSize;
-    private final int tcpReceiveBufferSize;
+    protected final boolean tcpNoDelay;
+    protected final boolean tcpKeepAlive;
+    protected final boolean reuseAddress;
+    protected final int tcpSendBufferSize;
+    protected final int tcpReceiveBufferSize;
 
     private NioGroup nioGroup;
-    private HttpChannelFactory channelFactory;
-    private final NioCorsConfig corsConfig;
+    private ChannelFactory<NioHttpServerChannel, NioHttpChannel> channelFactory;
 
     public NioHttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays,
                                   PageCacheRecycler pageCacheRecycler, ThreadPool threadPool, NamedXContentRegistry xContentRegistry,
-                                  HttpServerTransport.Dispatcher dispatcher) {
+                                  Dispatcher dispatcher) {
         super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher);
         this.pageCacheRecycler = pageCacheRecycler;
 
@@ -136,7 +135,7 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
             nioGroup = new NioGroup(daemonThreadFactory(this.settings, HTTP_SERVER_ACCEPTOR_THREAD_NAME_PREFIX), acceptorCount,
                 daemonThreadFactory(this.settings, HTTP_SERVER_WORKER_THREAD_NAME_PREFIX), workerCount,
                 (s) -> new EventHandler(this::onNonChannelException, s));
-            channelFactory = new HttpChannelFactory();
+            channelFactory = channelFactory();
             bindServer();
             success = true;
         } catch (IOException e) {
@@ -162,6 +161,10 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
         return nioGroup.bindServerChannel(socketAddress, channelFactory);
     }
 
+    protected ChannelFactory<NioHttpServerChannel, NioHttpChannel> channelFactory() {
+        return new HttpChannelFactory();
+    }
+
     static NioCorsConfig buildCorsConfig(Settings settings) {
         if (SETTING_CORS_ENABLED.get(settings) == false) {
             return NioCorsConfigBuilder.forOrigins().disable().build();
@@ -194,7 +197,7 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport {
             .build();
     }
 
-    private void acceptChannel(NioSocketChannel socketChannel) {
+    protected void acceptChannel(NioSocketChannel socketChannel) {
         super.serverAcceptedChannel((HttpChannel) socketChannel);
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 26ec50c0eb3..3115c08a946 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -200,11 +200,13 @@ import org.elasticsearch.xpack.security.rest.action.user.RestHasPrivilegesAction
 import org.elasticsearch.xpack.security.rest.action.user.RestPutUserAction;
 import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
+import org.elasticsearch.xpack.security.transport.SecurityHttpSettings;
 import org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport;
 import org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4ServerTransport;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
+import org.elasticsearch.xpack.security.transport.nio.SecurityNioHttpServerTransport;
 import org.elasticsearch.xpack.security.transport.nio.SecurityNioTransport;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -511,21 +513,22 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
 
             if (NetworkModule.HTTP_TYPE_SETTING.exists(settings)) {
                 final String httpType = NetworkModule.HTTP_TYPE_SETTING.get(settings);
-                if (httpType.equals(SecurityField.NAME4)) {
-                    SecurityNetty4HttpServerTransport.overrideSettings(builder, settings);
+                if (httpType.equals(SecurityField.NAME4) || httpType.equals(SecurityField.NIO)) {
+                    SecurityHttpSettings.overrideSettings(builder, settings);
                 } else {
                     final String message = String.format(
                             Locale.ROOT,
-                            "http type setting [%s] must be [%s] but is [%s]",
+                            "http type setting [%s] must be [%s] or [%s] but is [%s]",
                             NetworkModule.HTTP_TYPE_KEY,
                             SecurityField.NAME4,
+                            SecurityField.NIO,
                             httpType);
                     throw new IllegalArgumentException(message);
                 }
             } else {
                 // default to security4
                 builder.put(NetworkModule.HTTP_TYPE_KEY, SecurityField.NAME4);
-                SecurityNetty4HttpServerTransport.overrideSettings(builder, settings);
+                SecurityHttpSettings.overrideSettings(builder, settings);
             }
             builder.put(SecuritySettings.addUserSettings(settings));
             return builder.build();
@@ -869,8 +872,14 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         if (enabled == false) { // don't register anything if we are not enabled
             return Collections.emptyMap();
         }
-        return Collections.singletonMap(SecurityField.NAME4, () -> new SecurityNetty4HttpServerTransport(settings,
-                networkService, bigArrays, ipFilter.get(), getSslService(), threadPool, xContentRegistry, dispatcher));
+
+        Map<String, Supplier<HttpServerTransport>> httpTransports = new HashMap<>();
+        httpTransports.put(SecurityField.NAME4, () -> new SecurityNetty4HttpServerTransport(settings, networkService, bigArrays,
+            ipFilter.get(), getSslService(), threadPool, xContentRegistry, dispatcher));
+        httpTransports.put(SecurityField.NIO, () -> new SecurityNioHttpServerTransport(settings, networkService, bigArrays,
+            pageCacheRecycler, threadPool, xContentRegistry, dispatcher, ipFilter.get(), getSslService()));
+
+        return httpTransports;
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java
index 9109bb37e8c..8d304302e03 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java
@@ -5,8 +5,6 @@
  */
 package org.elasticsearch.xpack.security.rest;
 
-import io.netty.channel.Channel;
-import io.netty.handler.ssl.SslHandler;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
@@ -15,7 +13,6 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.HttpChannel;
-import org.elasticsearch.http.netty4.Netty4HttpChannel;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -24,7 +21,7 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequest.Method;
 import org.elasticsearch.xpack.core.security.rest.RestRequestFilter;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
-import org.elasticsearch.xpack.security.transport.ServerTransportFilter;
+import org.elasticsearch.xpack.security.transport.SSLEngineUtils;
 
 import java.io.IOException;
 
@@ -53,10 +50,7 @@ public class SecurityRestFilter implements RestHandler {
             // CORS - allow for preflight unauthenticated OPTIONS request
             if (extractClientCertificate) {
                 HttpChannel httpChannel = request.getHttpChannel();
-                Channel nettyChannel = ((Netty4HttpChannel) httpChannel).getNettyChannel();
-                SslHandler handler = nettyChannel.pipeline().get(SslHandler.class);
-                assert handler != null;
-                ServerTransportFilter.extractClientCertificates(logger, threadContext, handler.engine(), nettyChannel);
+                SSLEngineUtils.extractClientCertificates(logger, threadContext, httpChannel);
             }
             service.authenticate(maybeWrapRestRequest(request), ActionListener.wrap(
                 authentication -> {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SSLEngineUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SSLEngineUtils.java
new file mode 100644
index 00000000000..5bbcbaa0509
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SSLEngineUtils.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.transport;
+
+import io.netty.channel.Channel;
+import io.netty.handler.ssl.SslHandler;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.http.HttpChannel;
+import org.elasticsearch.http.netty4.Netty4HttpChannel;
+import org.elasticsearch.http.nio.NioHttpChannel;
+import org.elasticsearch.nio.SocketChannelContext;
+import org.elasticsearch.transport.TcpChannel;
+import org.elasticsearch.transport.netty4.Netty4TcpChannel;
+import org.elasticsearch.transport.nio.NioTcpChannel;
+import org.elasticsearch.xpack.security.authc.pki.PkiRealm;
+import org.elasticsearch.xpack.security.transport.nio.SSLChannelContext;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+
+public class SSLEngineUtils {
+
+    private SSLEngineUtils() {}
+
+    public static void extractClientCertificates(Logger logger, ThreadContext threadContext, HttpChannel httpChannel) {
+        SSLEngine sslEngine = getSSLEngine(httpChannel);
+        extract(logger, threadContext, sslEngine, httpChannel);
+    }
+
+    public static void extractClientCertificates(Logger logger, ThreadContext threadContext, TcpChannel tcpChannel) {
+        SSLEngine sslEngine = getSSLEngine(tcpChannel);
+        extract(logger, threadContext, sslEngine, tcpChannel);
+    }
+
+    public static SSLEngine getSSLEngine(HttpChannel httpChannel) {
+        if (httpChannel instanceof Netty4HttpChannel) {
+            Channel nettyChannel = ((Netty4HttpChannel) httpChannel).getNettyChannel();
+            SslHandler handler = nettyChannel.pipeline().get(SslHandler.class);
+            assert handler != null : "Must have SslHandler";
+            return handler.engine();
+        } else if (httpChannel instanceof NioHttpChannel) {
+            SocketChannelContext context = ((NioHttpChannel) httpChannel).getContext();
+            assert context instanceof SSLChannelContext : "Must be SSLChannelContext.class, found:  " + context.getClass();
+            return ((SSLChannelContext) context).getSSLEngine();
+        } else {
+            throw new AssertionError("Unknown channel class type: " + httpChannel.getClass());
+        }
+    }
+
+    public static SSLEngine getSSLEngine(TcpChannel tcpChannel) {
+        if (tcpChannel instanceof Netty4TcpChannel) {
+            Channel nettyChannel = ((Netty4TcpChannel) tcpChannel).getNettyChannel();
+            SslHandler handler = nettyChannel.pipeline().get(SslHandler.class);
+            assert handler != null : "Must have SslHandler";
+            return handler.engine();
+        } else if (tcpChannel instanceof NioTcpChannel) {
+            SocketChannelContext context = ((NioTcpChannel) tcpChannel).getContext();
+            assert context instanceof SSLChannelContext : "Must be SSLChannelContext.class, found:  " + context.getClass();
+            return ((SSLChannelContext) context).getSSLEngine();
+        } else {
+            throw new AssertionError("Unknown channel class type: " + tcpChannel.getClass());
+        }
+    }
+
+    private static void extract(Logger logger, ThreadContext threadContext, SSLEngine sslEngine, Object channel) {
+        try {
+            Certificate[] certs = sslEngine.getSession().getPeerCertificates();
+            if (certs instanceof X509Certificate[]) {
+                threadContext.putTransient(PkiRealm.PKI_CERT_HEADER_NAME, certs);
+            }
+        } catch (SSLPeerUnverifiedException e) {
+            // this happens when client authentication is optional and the client does not provide credentials. If client
+            // authentication was required then this connection should be closed before ever getting into this class
+            assert sslEngine.getNeedClientAuth() == false;
+            assert sslEngine.getWantClientAuth();
+            if (logger.isTraceEnabled()) {
+                logger.trace(
+                    (Supplier<?>) () -> new ParameterizedMessage(
+                        "SSL Peer did not present a certificate on channel [{}]", channel), e);
+            } else if (logger.isDebugEnabled()) {
+                logger.debug("SSL Peer did not present a certificate on channel [{}]", channel);
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpExceptionHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpExceptionHandler.java
new file mode 100644
index 00000000000..c1999c5ddfb
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpExceptionHandler.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.transport;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.common.component.Lifecycle;
+import org.elasticsearch.common.network.CloseableChannel;
+import org.elasticsearch.http.HttpChannel;
+
+import java.util.function.BiConsumer;
+
+import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isCloseDuringHandshakeException;
+import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isNotSslRecordException;
+import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isReceivedCertificateUnknownException;
+
+public final class SecurityHttpExceptionHandler implements BiConsumer<HttpChannel, Exception> {
+
+    private final Lifecycle lifecycle;
+    private final Logger logger;
+    private final BiConsumer<HttpChannel, Exception> fallback;
+
+    public SecurityHttpExceptionHandler(Logger logger, Lifecycle lifecycle, BiConsumer<HttpChannel, Exception> fallback) {
+        this.lifecycle = lifecycle;
+        this.logger = logger;
+        this.fallback = fallback;
+    }
+
+    public void accept(HttpChannel channel, Exception e) {
+        if (!lifecycle.started()) {
+            return;
+        }
+
+        if (isNotSslRecordException(e)) {
+            if (logger.isTraceEnabled()) {
+                logger.trace(new ParameterizedMessage("received plaintext http traffic on a https channel, closing connection {}",
+                    channel), e);
+            } else {
+                logger.warn("received plaintext http traffic on a https channel, closing connection {}", channel);
+            }
+            CloseableChannel.closeChannel(channel);
+        } else if (isCloseDuringHandshakeException(e)) {
+            if (logger.isTraceEnabled()) {
+                logger.trace(new ParameterizedMessage("connection {} closed during ssl handshake", channel), e);
+            } else {
+                logger.warn("connection {} closed during ssl handshake", channel);
+            }
+            CloseableChannel.closeChannel(channel);
+        } else if (isReceivedCertificateUnknownException(e)) {
+            if (logger.isTraceEnabled()) {
+                logger.trace(new ParameterizedMessage("http client did not trust server's certificate, closing connection {}",
+                    channel), e);
+            } else {
+                logger.warn("http client did not trust this server's certificate, closing connection {}", channel);
+            }
+            CloseableChannel.closeChannel(channel);
+        } else {
+            fallback.accept(channel, e);
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpSettings.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpSettings.java
new file mode 100644
index 00000000000..f8079535acf
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpSettings.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.transport;
+
+import org.elasticsearch.common.settings.Settings;
+
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION;
+import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
+
+public final class SecurityHttpSettings {
+
+    private SecurityHttpSettings() {}
+
+    public static void overrideSettings(Settings.Builder settingsBuilder, Settings settings) {
+        if (HTTP_SSL_ENABLED.get(settings) && SETTING_HTTP_COMPRESSION.exists(settings) == false) {
+            settingsBuilder.put(SETTING_HTTP_COMPRESSION.getKey(), false);
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
index 9427812ba13..2f0c40c1fdd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
@@ -5,11 +5,7 @@
  */
 package org.elasticsearch.xpack.security.transport;
 
-import io.netty.channel.Channel;
-import io.netty.handler.ssl.SslHandler;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.IndicesRequest;
@@ -20,11 +16,13 @@ import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.transport.TaskTransportChannel;
+import org.elasticsearch.transport.TcpChannel;
 import org.elasticsearch.transport.TcpTransportChannel;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.netty4.Netty4TcpChannel;
+import org.elasticsearch.transport.nio.NioTcpChannel;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
@@ -32,16 +30,10 @@ import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.action.SecurityActionMapper;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
-import org.elasticsearch.xpack.security.authc.pki.PkiRealm;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationUtils;
 
-import javax.net.ssl.SSLEngine;
-import javax.net.ssl.SSLPeerUnverifiedException;
-
 import java.io.IOException;
-import java.security.cert.Certificate;
-import java.security.cert.X509Certificate;
 
 import static org.elasticsearch.xpack.core.security.support.Exceptions.authenticationError;
 
@@ -115,13 +107,12 @@ public interface ServerTransportFilter {
                 unwrappedChannel = ((TaskTransportChannel) unwrappedChannel).getChannel();
             }
 
-            if (extractClientCert && (unwrappedChannel instanceof TcpTransportChannel) &&
-                ((TcpTransportChannel) unwrappedChannel).getChannel() instanceof Netty4TcpChannel) {
-                Channel channel = ((Netty4TcpChannel) ((TcpTransportChannel) unwrappedChannel).getChannel()).getLowLevelChannel();
-                SslHandler sslHandler = channel.pipeline().get(SslHandler.class);
-                if (channel.isOpen()) {
-                    assert sslHandler != null : "channel [" + channel + "] did not have a ssl handler. pipeline " + channel.pipeline();
-                    extractClientCertificates(logger, threadContext, sslHandler.engine(), channel);
+            if (extractClientCert && (unwrappedChannel instanceof TcpTransportChannel)) {
+                TcpChannel tcpChannel = ((TcpTransportChannel) unwrappedChannel).getChannel();
+                if (tcpChannel instanceof Netty4TcpChannel || tcpChannel instanceof NioTcpChannel) {
+                    if (tcpChannel.isOpen()) {
+                        SSLEngineUtils.extractClientCertificates(logger, threadContext, tcpChannel);
+                    }
                 }
             }
 
@@ -172,27 +163,6 @@ public interface ServerTransportFilter {
         }
     }
 
-    static void extractClientCertificates(Logger logger, ThreadContext threadContext, SSLEngine sslEngine, Channel channel) {
-        try {
-            Certificate[] certs = sslEngine.getSession().getPeerCertificates();
-            if (certs instanceof X509Certificate[]) {
-                threadContext.putTransient(PkiRealm.PKI_CERT_HEADER_NAME, certs);
-            }
-        } catch (SSLPeerUnverifiedException e) {
-            // this happens when client authentication is optional and the client does not provide credentials. If client
-            // authentication was required then this connection should be closed before ever getting into this class
-            assert sslEngine.getNeedClientAuth() == false;
-            assert sslEngine.getWantClientAuth();
-            if (logger.isTraceEnabled()) {
-                logger.trace(
-                        (Supplier<?>) () -> new ParameterizedMessage(
-                                "SSL Peer did not present a certificate on channel [{}]", channel), e);
-            } else if (logger.isDebugEnabled()) {
-                logger.debug("SSL Peer did not present a certificate on channel [{}]", channel);
-            }
-        }
-    }
-
     /**
      * A server transport filter rejects internal calls, which should be used on connections
      * where only clients connect to. This ensures that no client can send any internal actions
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransport.java
index d7a609f6f14..a728467f8bd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransport.java
@@ -8,8 +8,6 @@ package org.elasticsearch.xpack.security.transport.netty4;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandler;
 import io.netty.handler.ssl.SslHandler;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.common.network.CloseableChannel;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
@@ -19,18 +17,16 @@ import org.elasticsearch.http.netty4.Netty4HttpServerTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ssl.SSLConfiguration;
 import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.security.transport.SecurityHttpExceptionHandler;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 
 import javax.net.ssl.SSLEngine;
 
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION;
 import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
-import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isCloseDuringHandshakeException;
-import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isNotSslRecordException;
-import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isReceivedCertificateUnknownException;
 
 public class SecurityNetty4HttpServerTransport extends Netty4HttpServerTransport {
 
+    private final SecurityHttpExceptionHandler securityExceptionHandler;
     private final IPFilter ipFilter;
     private final SSLService sslService;
     private final SSLConfiguration sslConfiguration;
@@ -39,6 +35,7 @@ public class SecurityNetty4HttpServerTransport extends Netty4HttpServerTransport
                                              SSLService sslService, ThreadPool threadPool, NamedXContentRegistry xContentRegistry,
                                              Dispatcher dispatcher) {
         super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher);
+        this.securityExceptionHandler = new SecurityHttpExceptionHandler(logger, lifecycle, (c, e) -> super.onException(c, e));
         this.ipFilter = ipFilter;
         final boolean ssl = HTTP_SSL_ENABLED.get(settings);
         this.sslService = sslService;
@@ -51,41 +48,11 @@ public class SecurityNetty4HttpServerTransport extends Netty4HttpServerTransport
         } else {
             this.sslConfiguration = null;
         }
-
     }
 
     @Override
     protected void onException(HttpChannel channel, Exception e) {
-        if (!lifecycle.started()) {
-            return;
-        }
-
-        if (isNotSslRecordException(e)) {
-            if (logger.isTraceEnabled()) {
-                logger.trace(new ParameterizedMessage("received plaintext http traffic on a https channel, closing connection {}",
-                    channel), e);
-            } else {
-                logger.warn("received plaintext http traffic on a https channel, closing connection {}", channel);
-            }
-            CloseableChannel.closeChannel(channel);
-        } else if (isCloseDuringHandshakeException(e)) {
-            if (logger.isTraceEnabled()) {
-                logger.trace(new ParameterizedMessage("connection {} closed during ssl handshake", channel), e);
-            } else {
-                logger.warn("connection {} closed during ssl handshake", channel);
-            }
-            CloseableChannel.closeChannel(channel);
-        } else if (isReceivedCertificateUnknownException(e)) {
-            if (logger.isTraceEnabled()) {
-                logger.trace(new ParameterizedMessage("http client did not trust server's certificate, closing connection {}",
-                    channel), e);
-            } else {
-                logger.warn("http client did not trust this server's certificate, closing connection {}", channel);
-            }
-            CloseableChannel.closeChannel(channel);
-        } else {
-            super.onException(channel, e);
-        }
+        securityExceptionHandler.accept(channel, e);
     }
 
     @Override
@@ -115,10 +82,4 @@ public class SecurityNetty4HttpServerTransport extends Netty4HttpServerTransport
             ch.pipeline().addFirst("ip_filter", new IpFilterRemoteAddressFilter(ipFilter, IPFilter.HTTP_PROFILE_NAME));
         }
     }
-
-    public static void overrideSettings(Settings.Builder settingsBuilder, Settings settings) {
-        if (HTTP_SSL_ENABLED.get(settings) && SETTING_HTTP_COMPRESSION.exists(settings) == false) {
-            settingsBuilder.put(SETTING_HTTP_COMPRESSION.getKey(), false);
-        }
-    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilter.java
new file mode 100644
index 00000000000..afb13ceff2e
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilter.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.transport.nio;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.nio.NioSocketChannel;
+import org.elasticsearch.xpack.security.transport.filter.IPFilter;
+
+import java.util.function.Predicate;
+
+public final class NioIPFilter implements Predicate<NioSocketChannel> {
+
+    private final IPFilter filter;
+    private final String profile;
+
+    NioIPFilter(@Nullable IPFilter filter, String profile) {
+        this.filter = filter;
+        this.profile = profile;
+    }
+
+    @Override
+    public boolean test(NioSocketChannel nioChannel) {
+        if (filter != null) {
+            return filter.accept(profile, nioChannel.getRemoteAddress());
+        } else {
+            return true;
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java
index da348ea1f78..c83bd16ca95 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java
@@ -14,6 +14,7 @@ import org.elasticsearch.nio.SocketChannelContext;
 import org.elasticsearch.nio.NioSelector;
 import org.elasticsearch.nio.WriteOperation;
 
+import javax.net.ssl.SSLEngine;
 import java.io.IOException;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
@@ -164,6 +165,10 @@ public final class SSLChannelContext extends SocketChannelContext {
         }
     }
 
+    public SSLEngine getSSLEngine() {
+        return sslDriver.getSSLEngine();
+    }
+
     private static class CloseNotifyOperation implements WriteOperation {
 
         private static final BiConsumer<Void, Exception> LISTENER = (v, t) -> {};
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLDriver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLDriver.java
index 4080574713c..382230684c7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLDriver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLDriver.java
@@ -96,6 +96,10 @@ public class SSLDriver implements AutoCloseable {
         }
     }
 
+    public SSLEngine getSSLEngine() {
+        return engine;
+    }
+
     public boolean hasFlushPending() {
         return networkWriteBuffer.hasRemaining();
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
new file mode 100644
index 00000000000..006c78b4ae0
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.transport.nio;
+
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.recycler.Recycler;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.http.nio.HttpReadWriteHandler;
+import org.elasticsearch.http.nio.NioHttpChannel;
+import org.elasticsearch.http.nio.NioHttpServerChannel;
+import org.elasticsearch.http.nio.NioHttpServerTransport;
+import org.elasticsearch.nio.BytesChannelContext;
+import org.elasticsearch.nio.ChannelFactory;
+import org.elasticsearch.nio.InboundChannelBuffer;
+import org.elasticsearch.nio.NioSelector;
+import org.elasticsearch.nio.NioSocketChannel;
+import org.elasticsearch.nio.ServerChannelContext;
+import org.elasticsearch.nio.SocketChannelContext;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.ssl.SSLConfiguration;
+import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.security.transport.SecurityHttpExceptionHandler;
+import org.elasticsearch.xpack.security.transport.filter.IPFilter;
+
+import javax.net.ssl.SSLEngine;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
+
+public class SecurityNioHttpServerTransport extends NioHttpServerTransport {
+
+    private final SecurityHttpExceptionHandler securityExceptionHandler;
+    private final IPFilter ipFilter;
+    private final NioIPFilter nioIpFilter;
+    private final SSLService sslService;
+    private final SSLConfiguration sslConfiguration;
+    private final boolean sslEnabled;
+
+    public SecurityNioHttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays,
+                                          PageCacheRecycler pageCacheRecycler, ThreadPool threadPool,
+                                          NamedXContentRegistry xContentRegistry, Dispatcher dispatcher, IPFilter ipFilter,
+                                          SSLService sslService) {
+        super(settings, networkService, bigArrays, pageCacheRecycler, threadPool, xContentRegistry, dispatcher);
+        this.securityExceptionHandler = new SecurityHttpExceptionHandler(logger, lifecycle, (c, e) -> super.onException(c, e));
+        this.ipFilter = ipFilter;
+        this.nioIpFilter = new NioIPFilter(ipFilter, IPFilter.HTTP_PROFILE_NAME);
+        this.sslEnabled = HTTP_SSL_ENABLED.get(settings);
+        this.sslService = sslService;
+        if (sslEnabled) {
+            this.sslConfiguration = sslService.sslConfiguration(SSLService.getHttpTransportSSLSettings(settings), Settings.EMPTY);
+            if (sslService.isConfigurationValidForServerUsage(sslConfiguration) == false) {
+                throw new IllegalArgumentException("a key must be provided to run as a server. the key should be configured using the " +
+                    "[xpack.security.http.ssl.key] or [xpack.security.http.ssl.keystore.path] setting");
+            }
+        } else {
+            this.sslConfiguration = null;
+        }
+    }
+
+    @Override
+    protected void doStart() {
+        super.doStart();
+        ipFilter.setBoundHttpTransportAddress(this.boundAddress());
+    }
+
+    protected SecurityHttpChannelFactory channelFactory() {
+        return new SecurityHttpChannelFactory();
+    }
+
+    class SecurityHttpChannelFactory extends ChannelFactory<NioHttpServerChannel, NioHttpChannel> {
+
+        private SecurityHttpChannelFactory() {
+            super(new RawChannelFactory(tcpNoDelay, tcpKeepAlive, reuseAddress, tcpSendBufferSize, tcpReceiveBufferSize));
+        }
+
+        @Override
+        public NioHttpChannel createChannel(NioSelector selector, SocketChannel channel) throws IOException {
+            NioHttpChannel httpChannel = new NioHttpChannel(channel);
+            Supplier<InboundChannelBuffer.Page> pageSupplier = () -> {
+                Recycler.V<byte[]> bytes = pageCacheRecycler.bytePage(false);
+                return new InboundChannelBuffer.Page(ByteBuffer.wrap(bytes.v()), bytes::close);
+            };
+            HttpReadWriteHandler httpHandler = new HttpReadWriteHandler(httpChannel,SecurityNioHttpServerTransport.this,
+                handlingSettings, corsConfig);
+            InboundChannelBuffer buffer = new InboundChannelBuffer(pageSupplier);
+            Consumer<Exception> exceptionHandler = (e) -> securityExceptionHandler.accept(httpChannel, e);
+
+            SocketChannelContext context;
+            if (sslEnabled) {
+                SSLEngine sslEngine;
+                boolean hostnameVerificationEnabled = sslConfiguration.verificationMode().isHostnameVerificationEnabled();
+                if (hostnameVerificationEnabled) {
+                    InetSocketAddress address = (InetSocketAddress) channel.getRemoteAddress();
+                    // we create the socket based on the name given. don't reverse DNS
+                    sslEngine = sslService.createSSLEngine(sslConfiguration, address.getHostString(), address.getPort());
+                } else {
+                    sslEngine = sslService.createSSLEngine(sslConfiguration, null, -1);
+                }
+                SSLDriver sslDriver = new SSLDriver(sslEngine, false);
+                context = new SSLChannelContext(httpChannel, selector, exceptionHandler, sslDriver, httpHandler, buffer, nioIpFilter);
+            } else {
+                context = new BytesChannelContext(httpChannel, selector, exceptionHandler, httpHandler, buffer, nioIpFilter);
+            }
+            httpChannel.setContext(context);
+
+            return httpChannel;
+        }
+
+        @Override
+        public NioHttpServerChannel createServerChannel(NioSelector selector, ServerSocketChannel channel) {
+            NioHttpServerChannel httpServerChannel = new NioHttpServerChannel(channel);
+            Consumer<Exception> exceptionHandler = (e) -> onServerException(httpServerChannel, e);
+            Consumer<NioSocketChannel> acceptor = SecurityNioHttpServerTransport.this::acceptChannel;
+            ServerChannelContext context = new ServerChannelContext(httpServerChannel, this, selector, acceptor, exceptionHandler);
+            httpServerChannel.setContext(context);
+
+            return httpServerChannel;
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java
index fb94b669e83..71e14696a11 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java
@@ -44,7 +44,6 @@ import java.nio.channels.SocketChannel;
 import java.util.Collections;
 import java.util.Map;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.xpack.core.security.SecurityField.setting;
@@ -129,19 +128,11 @@ public class SecurityNioTransport extends NioTransport {
         return new SecurityTcpChannelFactory(profileSettings, isClient);
     }
 
-    private boolean validateChannel(NioSocketChannel channel) {
-        if (authenticator != null) {
-            NioTcpChannel nioTcpChannel = (NioTcpChannel) channel;
-            return authenticator.accept(nioTcpChannel.getProfile(), nioTcpChannel.getRemoteAddress());
-        } else {
-            return true;
-        }
-    }
-
     private class SecurityTcpChannelFactory extends TcpChannelFactory {
 
         private final String profileName;
         private final boolean isClient;
+        private final NioIPFilter ipFilter;
 
         private SecurityTcpChannelFactory(ProfileSettings profileSettings, boolean isClient) {
             super(new RawChannelFactory(profileSettings.tcpNoDelay,
@@ -151,12 +142,12 @@ public class SecurityNioTransport extends NioTransport {
                 Math.toIntExact(profileSettings.receiveBufferSize.getBytes())));
             this.profileName = profileSettings.profileName;
             this.isClient = isClient;
+            this.ipFilter = new NioIPFilter(authenticator, profileName);
         }
 
         @Override
         public NioTcpChannel createChannel(NioSelector selector, SocketChannel channel) throws IOException {
             NioTcpChannel nioChannel = new NioTcpChannel(profileName, channel);
-            SocketChannelContext context;
             Supplier<InboundChannelBuffer.Page> pageSupplier = () -> {
                 Recycler.V<byte[]> bytes = pageCacheRecycler.bytePage(false);
                 return new InboundChannelBuffer.Page(ByteBuffer.wrap(bytes.v()), bytes::close);
@@ -164,8 +155,8 @@ public class SecurityNioTransport extends NioTransport {
             TcpReadWriteHandler readWriteHandler = new TcpReadWriteHandler(nioChannel, SecurityNioTransport.this);
             InboundChannelBuffer buffer = new InboundChannelBuffer(pageSupplier);
             Consumer<Exception> exceptionHandler = (e) -> onException(nioChannel, e);
-            Predicate<NioSocketChannel> filter = SecurityNioTransport.this::validateChannel;
 
+            SocketChannelContext context;
             if (sslEnabled) {
                 SSLEngine sslEngine;
                 SSLConfiguration defaultConfig = profileConfiguration.get(TcpTransport.DEFAULT_PROFILE);
@@ -179,9 +170,9 @@ public class SecurityNioTransport extends NioTransport {
                     sslEngine = sslService.createSSLEngine(sslConfig, null, -1);
                 }
                 SSLDriver sslDriver = new SSLDriver(sslEngine, isClient);
-                context = new SSLChannelContext(nioChannel, selector, exceptionHandler, sslDriver, readWriteHandler, buffer, filter);
+                context = new SSLChannelContext(nioChannel, selector, exceptionHandler, sslDriver, readWriteHandler, buffer, ipFilter);
             } else {
-                context = new BytesChannelContext(nioChannel, selector, exceptionHandler, readWriteHandler, buffer, filter);
+                context = new BytesChannelContext(nioChannel, selector, exceptionHandler, readWriteHandler, buffer, ipFilter);
             }
             nioChannel.setContext(context);
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java
index e6db3407496..9bb0e44eb66 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java
@@ -244,6 +244,7 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase {
         builder.put(customSettings, false); // handle secure settings separately
         builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         builder.put(NetworkModule.TRANSPORT_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO);
+        builder.put(NetworkModule.HTTP_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO);
         Settings.Builder customBuilder = Settings.builder().put(customSettings);
         if (customBuilder.getSecureSettings() != null) {
             SecuritySettingsSource.addSecureSettings(builder, secureSettings ->
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
index 2e0662264a2..df1456c3790 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
@@ -126,6 +126,7 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
         Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal))
                 .put(XPackSettings.SECURITY_ENABLED.getKey(), true)
                 .put(NetworkModule.TRANSPORT_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO)
+                .put(NetworkModule.HTTP_TYPE_KEY, randomBoolean() ? SecurityField.NAME4 : SecurityField.NIO)
                 //TODO: for now isolate security tests from watcher & monitoring (randomize this later)
                 .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
                 .put(XPackSettings.MONITORING_ENABLED.getKey(), false)
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityHttpSettingsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityHttpSettingsTests.java
new file mode 100644
index 00000000000..56c79a4c127
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityHttpSettingsTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.transport;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+import static org.hamcrest.Matchers.is;
+
+public class SecurityHttpSettingsTests extends ESTestCase {
+
+    public void testDisablesCompressionByDefaultForSsl() {
+        Settings settings = Settings.builder()
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true).build();
+
+        Settings.Builder pluginSettingsBuilder = Settings.builder();
+        SecurityHttpSettings.overrideSettings(pluginSettingsBuilder, settings);
+        assertThat(HttpTransportSettings.SETTING_HTTP_COMPRESSION.get(pluginSettingsBuilder.build()), is(false));
+    }
+
+    public void testLeavesCompressionOnIfNotSsl() {
+        Settings settings = Settings.builder()
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), false).build();
+        Settings.Builder pluginSettingsBuilder = Settings.builder();
+        SecurityHttpSettings.overrideSettings(pluginSettingsBuilder, settings);
+        assertThat(pluginSettingsBuilder.build().isEmpty(), is(true));
+    }
+
+    public void testDoesNotChangeExplicitlySetCompression() {
+        Settings settings = Settings.builder()
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
+            .put(HttpTransportSettings.SETTING_HTTP_COMPRESSION.getKey(), true)
+            .build();
+
+        Settings.Builder pluginSettingsBuilder = Settings.builder();
+        SecurityHttpSettings.overrideSettings(pluginSettingsBuilder, settings);
+        assertThat(pluginSettingsBuilder.build().isEmpty(), is(true));
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
index ec925f43abe..ad64dea79a5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
@@ -14,7 +14,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.http.NullDispatcher;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -144,34 +143,6 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
         assertThat(customEngine.getEnabledProtocols(), not(equalTo(defaultEngine.getEnabledProtocols())));
     }
 
-    public void testDisablesCompressionByDefaultForSsl() throws Exception {
-        Settings settings = Settings.builder()
-                .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true).build();
-
-        Settings.Builder pluginSettingsBuilder = Settings.builder();
-        SecurityNetty4HttpServerTransport.overrideSettings(pluginSettingsBuilder, settings);
-        assertThat(HttpTransportSettings.SETTING_HTTP_COMPRESSION.get(pluginSettingsBuilder.build()), is(false));
-    }
-
-    public void testLeavesCompressionOnIfNotSsl() throws Exception {
-        Settings settings = Settings.builder()
-                .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), false).build();
-        Settings.Builder pluginSettingsBuilder = Settings.builder();
-        SecurityNetty4HttpServerTransport.overrideSettings(pluginSettingsBuilder, settings);
-        assertThat(pluginSettingsBuilder.build().isEmpty(), is(true));
-    }
-
-    public void testDoesNotChangeExplicitlySetCompression() throws Exception {
-        Settings settings = Settings.builder()
-                .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
-                .put(HttpTransportSettings.SETTING_HTTP_COMPRESSION.getKey(), true)
-                .build();
-
-        Settings.Builder pluginSettingsBuilder = Settings.builder();
-        SecurityNetty4HttpServerTransport.overrideSettings(pluginSettingsBuilder, settings);
-        assertThat(pluginSettingsBuilder.build().isEmpty(), is(true));
-    }
-
     public void testThatExceptionIsThrownWhenConfiguredWithoutSslKey() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilterTests.java
new file mode 100644
index 00000000000..1832669fce1
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilterTests.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.transport.nio;
+
+import org.elasticsearch.common.component.Lifecycle;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.nio.NioSocketChannel;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.xpack.security.audit.AuditTrailService;
+import org.elasticsearch.xpack.security.transport.filter.IPFilter;
+import org.junit.Before;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class NioIPFilterTests extends ESTestCase {
+
+    private NioIPFilter nioIPFilter;
+
+    @Before
+    public void init() throws Exception {
+        Settings settings = Settings.builder()
+            .put("xpack.security.transport.filter.allow", "127.0.0.1")
+            .put("xpack.security.transport.filter.deny", "10.0.0.0/8")
+            .build();
+
+        boolean isHttpEnabled = randomBoolean();
+
+        Transport transport = mock(Transport.class);
+        TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 9300);
+        when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { address }, address));
+        when(transport.lifecycleState()).thenReturn(Lifecycle.State.STARTED);
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(
+            IPFilter.HTTP_FILTER_ALLOW_SETTING,
+            IPFilter.HTTP_FILTER_DENY_SETTING,
+            IPFilter.IP_FILTER_ENABLED_HTTP_SETTING,
+            IPFilter.IP_FILTER_ENABLED_SETTING,
+            IPFilter.TRANSPORT_FILTER_ALLOW_SETTING,
+            IPFilter.TRANSPORT_FILTER_DENY_SETTING,
+            IPFilter.PROFILE_FILTER_ALLOW_SETTING,
+            IPFilter.PROFILE_FILTER_DENY_SETTING)));
+        XPackLicenseState licenseState = mock(XPackLicenseState.class);
+        when(licenseState.isIpFilteringAllowed()).thenReturn(true);
+        when(licenseState.isSecurityEnabled()).thenReturn(true);
+        AuditTrailService auditTrailService = new AuditTrailService(settings, Collections.emptyList(), licenseState);
+        IPFilter ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
+        ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
+        if (isHttpEnabled) {
+            HttpServerTransport httpTransport = mock(HttpServerTransport.class);
+            TransportAddress httpAddress = new TransportAddress(InetAddress.getLoopbackAddress(), 9200);
+            when(httpTransport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { httpAddress }, httpAddress));
+            when(httpTransport.lifecycleState()).thenReturn(Lifecycle.State.STARTED);
+            ipFilter.setBoundHttpTransportAddress(httpTransport.boundAddress());
+        }
+
+        if (isHttpEnabled) {
+            nioIPFilter = new NioIPFilter(ipFilter, IPFilter.HTTP_PROFILE_NAME);
+        } else {
+            nioIPFilter = new NioIPFilter(ipFilter, "default");
+        }
+    }
+
+    public void testThatFilteringWorksByIp() throws Exception {
+        InetSocketAddress localhostAddr = new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 12345);
+        NioSocketChannel channel1 = mock(NioSocketChannel.class);
+        when(channel1.getRemoteAddress()).thenReturn(localhostAddr);
+        assertThat(nioIPFilter.test(channel1), is(true));
+
+        InetSocketAddress remoteAddr = new InetSocketAddress(InetAddresses.forString("10.0.0.8"), 12345);
+        NioSocketChannel channel2 = mock(NioSocketChannel.class);
+        when(channel2.getRemoteAddress()).thenReturn(remoteAddr);
+        assertThat(nioIPFilter.test(channel2), is(false));
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransportTests.java
new file mode 100644
index 00000000000..b5d84d45916
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransportTests.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.transport.nio;
+
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.http.NullDispatcher;
+import org.elasticsearch.http.nio.NioHttpChannel;
+import org.elasticsearch.nio.NioSelector;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.ssl.SSLClientAuth;
+import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.security.transport.SSLEngineUtils;
+import org.elasticsearch.xpack.security.transport.filter.IPFilter;
+import org.junit.Before;
+
+import javax.net.ssl.SSLEngine;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.channels.SocketChannel;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Locale;
+
+import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SecurityNioHttpServerTransportTests extends ESTestCase {
+
+    private SSLService sslService;
+    private Environment env;
+    private InetSocketAddress address = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0);
+
+    @Before
+    public void createSSLService() {
+        Path testNodeStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        Settings settings = Settings.builder()
+            .put("xpack.ssl.keystore.path", testNodeStore)
+            .put("path.home", createTempDir())
+            .setSecureSettings(secureSettings)
+            .build();
+        env = TestEnvironment.newEnvironment(settings);
+        sslService = new SSLService(settings, env);
+    }
+
+    public void testDefaultClientAuth() throws IOException {
+        Settings settings = Settings.builder()
+            .put(env.settings())
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true).build();
+        sslService = new SSLService(settings, env);
+        SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
+            new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
+            xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService);
+        SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
+        SocketChannel socketChannel = mock(SocketChannel.class);
+        when(socketChannel.getRemoteAddress()).thenReturn(address);
+        NioHttpChannel channel = factory.createChannel(mock(NioSelector.class), socketChannel);
+        SSLEngine engine = SSLEngineUtils.getSSLEngine(channel);
+
+        assertThat(engine.getNeedClientAuth(), is(false));
+        assertThat(engine.getWantClientAuth(), is(false));
+    }
+
+    public void testOptionalClientAuth() throws IOException {
+        String value = randomFrom(SSLClientAuth.OPTIONAL.name(), SSLClientAuth.OPTIONAL.name().toLowerCase(Locale.ROOT));
+        Settings settings = Settings.builder()
+            .put(env.settings())
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
+            .put("xpack.security.http.ssl.client_authentication", value).build();
+        sslService = new SSLService(settings, env);
+        SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
+            new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
+            xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService);
+
+        SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
+        SocketChannel socketChannel = mock(SocketChannel.class);
+        when(socketChannel.getRemoteAddress()).thenReturn(address);
+        NioHttpChannel channel = factory.createChannel(mock(NioSelector.class), socketChannel);
+        SSLEngine engine = SSLEngineUtils.getSSLEngine(channel);
+        assertThat(engine.getNeedClientAuth(), is(false));
+        assertThat(engine.getWantClientAuth(), is(true));
+    }
+
+    public void testRequiredClientAuth() throws IOException {
+        String value = randomFrom(SSLClientAuth.REQUIRED.name(), SSLClientAuth.REQUIRED.name().toLowerCase(Locale.ROOT));
+        Settings settings = Settings.builder()
+            .put(env.settings())
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
+            .put("xpack.security.http.ssl.client_authentication", value).build();
+        sslService = new SSLService(settings, env);
+        SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
+            new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
+            xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService);
+
+        SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
+        SocketChannel socketChannel = mock(SocketChannel.class);
+        when(socketChannel.getRemoteAddress()).thenReturn(address);
+        NioHttpChannel channel = factory.createChannel(mock(NioSelector.class), socketChannel);
+        SSLEngine engine = SSLEngineUtils.getSSLEngine(channel);
+        assertThat(engine.getNeedClientAuth(), is(true));
+        assertThat(engine.getWantClientAuth(), is(false));
+    }
+
+    public void testNoClientAuth() throws IOException {
+        String value = randomFrom(SSLClientAuth.NONE.name(), SSLClientAuth.NONE.name().toLowerCase(Locale.ROOT));
+        Settings settings = Settings.builder()
+            .put(env.settings())
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
+            .put("xpack.security.http.ssl.client_authentication", value).build();
+        sslService = new SSLService(settings, env);
+        SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
+            new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
+            xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService);
+
+        SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
+        SocketChannel socketChannel = mock(SocketChannel.class);
+        when(socketChannel.getRemoteAddress()).thenReturn(address);
+        NioHttpChannel channel = factory.createChannel(mock(NioSelector.class), socketChannel);
+        SSLEngine engine = SSLEngineUtils.getSSLEngine(channel);
+        assertThat(engine.getNeedClientAuth(), is(false));
+        assertThat(engine.getWantClientAuth(), is(false));
+    }
+
+    public void testCustomSSLConfiguration() throws IOException {
+        Settings settings = Settings.builder()
+            .put(env.settings())
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true).build();
+        sslService = new SSLService(settings, env);
+        SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
+            new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
+            xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService);
+        SecurityNioHttpServerTransport.SecurityHttpChannelFactory factory = transport.channelFactory();
+        SocketChannel socketChannel = mock(SocketChannel.class);
+        when(socketChannel.getRemoteAddress()).thenReturn(address);
+        NioHttpChannel channel = factory.createChannel(mock(NioSelector.class), socketChannel);
+        SSLEngine defaultEngine = SSLEngineUtils.getSSLEngine(channel);
+
+        settings = Settings.builder()
+            .put(env.settings())
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
+            .put("xpack.security.http.ssl.supported_protocols", "TLSv1.2")
+            .build();
+        sslService = new SSLService(settings, TestEnvironment.newEnvironment(settings));
+        transport = new SecurityNioHttpServerTransport(settings,
+            new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
+            xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService);
+        factory = transport.channelFactory();
+        channel = factory.createChannel(mock(NioSelector.class), socketChannel);
+        SSLEngine customEngine = SSLEngineUtils.getSSLEngine(channel);
+        assertThat(customEngine.getEnabledProtocols(), arrayContaining("TLSv1.2"));
+        assertThat(customEngine.getEnabledProtocols(), not(equalTo(defaultEngine.getEnabledProtocols())));
+    }
+
+    public void testThatExceptionIsThrownWhenConfiguredWithoutSslKey() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
+        Settings settings = Settings.builder()
+            .put("xpack.ssl.truststore.path",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"))
+            .setSecureSettings(secureSettings)
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
+            .put("path.home", createTempDir())
+            .build();
+        env = TestEnvironment.newEnvironment(settings);
+        sslService = new SSLService(settings, env);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new SecurityNioHttpServerTransport(settings,
+                new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
+                xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService));
+        assertThat(e.getMessage(), containsString("key must be provided"));
+    }
+
+    public void testNoExceptionWhenConfiguredWithoutSslKeySSLDisabled() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
+        Settings settings = Settings.builder()
+            .put("xpack.ssl.truststore.path",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"))
+            .setSecureSettings(secureSettings)
+            .put("path.home", createTempDir())
+            .build();
+        env = TestEnvironment.newEnvironment(settings);
+        sslService = new SSLService(settings, env);
+        SecurityNioHttpServerTransport transport = new SecurityNioHttpServerTransport(settings,
+            new NetworkService(Collections.emptyList()), mock(BigArrays.class), mock(PageCacheRecycler.class), mock(ThreadPool.class),
+            xContentRegistry(), new NullDispatcher(), mock(IPFilter.class), sslService);
+    }
+}

From a612404b1f84ce92a24a69b9ad0b538f4c8f3428 Mon Sep 17 00:00:00 2001
From: Tim Brooks <tim@uncontended.net>
Date: Fri, 13 Jul 2018 23:37:15 -0600
Subject: [PATCH 022/260] Fix compile issues introduced by merge (#32058)

The build was broken due to some issues with the merging of #32018. A
method that was public went private before the PR was merged. That did
not cause a merge conflict (so the PR was merged successfully). But it
did cause the build to fail.
---
 .../security/transport/nio/SecurityNioHttpServerTransport.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
index 006c78b4ae0..50a78d93c71 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
@@ -59,7 +59,7 @@ public class SecurityNioHttpServerTransport extends NioHttpServerTransport {
         this.sslEnabled = HTTP_SSL_ENABLED.get(settings);
         this.sslService = sslService;
         if (sslEnabled) {
-            this.sslConfiguration = sslService.sslConfiguration(SSLService.getHttpTransportSSLSettings(settings), Settings.EMPTY);
+            this.sslConfiguration = sslService.getHttpTransportSSLConfiguration();
             if (sslService.isConfigurationValidForServerUsage(sslConfiguration) == false) {
                 throw new IllegalArgumentException("a key must be provided to run as a server. the key should be configured using the " +
                     "[xpack.security.http.ssl.key] or [xpack.security.http.ssl.keystore.path] setting");

From ccf61264101437c1d188c6dec24fc9348dfe0a63 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Sat, 14 Jul 2018 09:03:35 +0200
Subject: [PATCH 023/260] SCRIPTING: Remove unused
 MultiSearchTemplateRequestBuilder (#32049)

* Ever since 46e8d97813addd8c57fa54d2c700d26a171f2dbb this class is unused
---
 .../MultiSearchTemplateRequestBuilder.java    | 65 -------------------
 1 file changed, 65 deletions(-)
 delete mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java

diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java
deleted file mode 100644
index c4dac0dd88e..00000000000
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestBuilder.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.mustache;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.ElasticsearchClient;
-
-public class MultiSearchTemplateRequestBuilder
-        extends ActionRequestBuilder<MultiSearchTemplateRequest, MultiSearchTemplateResponse> {
-
-    protected MultiSearchTemplateRequestBuilder(ElasticsearchClient client, MultiSearchTemplateAction action) {
-        super(client, action, new MultiSearchTemplateRequest());
-    }
-
-    public MultiSearchTemplateRequestBuilder add(SearchTemplateRequest request) {
-        if (request.getRequest().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed()
-                && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
-            request.getRequest().indicesOptions(request().indicesOptions());
-        }
-
-        super.request.add(request);
-        return this;
-    }
-
-    public MultiSearchTemplateRequestBuilder add(SearchTemplateRequestBuilder request) {
-        if (request.request().getRequest().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed()
-                && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
-            request.request().getRequest().indicesOptions(request().indicesOptions());
-        }
-
-        super.request.add(request);
-        return this;
-    }
-
-    public MultiSearchTemplateRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
-        request().indicesOptions(indicesOptions);
-        return this;
-    }
-
-    /**
-     * Sets how many search requests specified in this multi search requests are allowed to be ran concurrently.
-     */
-    public MultiSearchTemplateRequestBuilder setMaxConcurrentSearchRequests(int maxConcurrentSearchRequests) {
-        request().maxConcurrentSearchRequests(maxConcurrentSearchRequests);
-        return this;
-    }
-}

From b65c586cef0c8b46aaab26bc3ea9ef81c7653aa9 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Sat, 14 Jul 2018 13:37:59 +0200
Subject: [PATCH 024/260] Cleanup Duplication in `PainlessScriptEngine`
 (#31991)

* Cleanup Duplication in `PainlessScriptEngine`
* Extract duplicate building of compiler settings to method
* Remove dead method params + dead constant in `ScriptProcessor`
---
 .../ingest/common/ScriptProcessor.java        |  3 -
 .../painless/PainlessScriptEngine.java        | 85 ++++++-------------
 2 files changed, 26 insertions(+), 62 deletions(-)

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index 74c68fd5c26..169b2ab646a 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.ingest.common;
 
-import com.fasterxml.jackson.core.JsonFactory;
-
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -48,7 +46,6 @@ import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationExcept
 public final class ScriptProcessor extends AbstractProcessor {
 
     public static final String TYPE = "script";
-    private static final JsonFactory JSON_FACTORY = new JsonFactory();
 
     private final Script script;
     private final ScriptService scriptService;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
index ae1944c9bd3..4560fd85a65 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
@@ -366,44 +366,7 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
     }
 
     Object compile(Compiler compiler, String scriptName, String source, Map<String, String> params, Object... args) {
-        final CompilerSettings compilerSettings;
-
-        if (params.isEmpty()) {
-            // Use the default settings.
-            compilerSettings = defaultCompilerSettings;
-        } else {
-            // Use custom settings specified by params.
-            compilerSettings = new CompilerSettings();
-
-            // Except regexes enabled - this is a node level setting and can't be changed in the request.
-            compilerSettings.setRegexesEnabled(defaultCompilerSettings.areRegexesEnabled());
-
-            Map<String, String> copy = new HashMap<>(params);
-
-            String value = copy.remove(CompilerSettings.MAX_LOOP_COUNTER);
-            if (value != null) {
-                compilerSettings.setMaxLoopCounter(Integer.parseInt(value));
-            }
-
-            value = copy.remove(CompilerSettings.PICKY);
-            if (value != null) {
-                compilerSettings.setPicky(Boolean.parseBoolean(value));
-            }
-
-            value = copy.remove(CompilerSettings.INITIAL_CALL_SITE_DEPTH);
-            if (value != null) {
-                compilerSettings.setInitialCallSiteDepth(Integer.parseInt(value));
-            }
-
-            value = copy.remove(CompilerSettings.REGEX_ENABLED.getKey());
-            if (value != null) {
-                throw new IllegalArgumentException("[painless.regex.enabled] can only be set on node startup.");
-            }
-
-            if (!copy.isEmpty()) {
-                throw new IllegalArgumentException("Unrecognized compile-time parameter(s): " + copy);
-            }
-        }
+        final CompilerSettings compilerSettings = buildCompilerSettings(params);
 
         // Check we ourselves are not being called by unprivileged code.
         SpecialPermission.check();
@@ -434,14 +397,33 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
             }, COMPILATION_CONTEXT);
         // Note that it is safe to catch any of the following errors since Painless is stateless.
         } catch (OutOfMemoryError | StackOverflowError | VerifyError | Exception e) {
-            throw convertToScriptException(scriptName == null ? source : scriptName, source, e);
+            throw convertToScriptException(source, e);
         }
     }
 
     void compile(Compiler compiler, Loader loader, MainMethodReserved reserved,
                  String scriptName, String source, Map<String, String> params) {
-        final CompilerSettings compilerSettings;
+        final CompilerSettings compilerSettings = buildCompilerSettings(params);
 
+        try {
+            // Drop all permissions to actually compile the code itself.
+            AccessController.doPrivileged(new PrivilegedAction<Void>() {
+                @Override
+                public Void run() {
+                    String name = scriptName == null ? source : scriptName;
+                    compiler.compile(loader, reserved, name, source, compilerSettings);
+
+                    return null;
+                }
+            }, COMPILATION_CONTEXT);
+            // Note that it is safe to catch any of the following errors since Painless is stateless.
+        } catch (OutOfMemoryError | StackOverflowError | VerifyError | Exception e) {
+            throw convertToScriptException(source, e);
+        }
+    }
+
+    private CompilerSettings buildCompilerSettings(Map<String, String> params) {
+        CompilerSettings compilerSettings;
         if (params.isEmpty()) {
             // Use the default settings.
             compilerSettings = defaultCompilerSettings;
@@ -478,25 +460,10 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
                 throw new IllegalArgumentException("Unrecognized compile-time parameter(s): " + copy);
             }
         }
-
-        try {
-            // Drop all permissions to actually compile the code itself.
-            AccessController.doPrivileged(new PrivilegedAction<Void>() {
-                @Override
-                public Void run() {
-                    String name = scriptName == null ? source : scriptName;
-                    compiler.compile(loader, reserved, name, source, compilerSettings);
-
-                    return null;
-                }
-            }, COMPILATION_CONTEXT);
-            // Note that it is safe to catch any of the following errors since Painless is stateless.
-        } catch (OutOfMemoryError | StackOverflowError | VerifyError | Exception e) {
-            throw convertToScriptException(scriptName == null ? source : scriptName, source, e);
-        }
+        return compilerSettings;
     }
 
-    private ScriptException convertToScriptException(String scriptName, String scriptSource, Throwable t) {
+    private ScriptException convertToScriptException(String scriptSource, Throwable t) {
         // create a script stack: this is just the script portion
         List<String> scriptStack = new ArrayList<>();
         for (StackTraceElement element : t.getStackTrace()) {
@@ -507,7 +474,7 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
                     scriptStack.add("<<< unknown portion of script >>>");
                 } else {
                     offset--; // offset is 1 based, line numbers must be!
-                    int startOffset = getPreviousStatement(scriptSource, offset);
+                    int startOffset = getPreviousStatement(offset);
                     int endOffset = getNextStatement(scriptSource, offset);
                     StringBuilder snippet = new StringBuilder();
                     if (startOffset > 0) {
@@ -535,7 +502,7 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
     }
 
     // very simple heuristic: +/- 25 chars. can be improved later.
-    private int getPreviousStatement(String scriptSource, int offset) {
+    private int getPreviousStatement(int offset) {
         return Math.max(0, offset - 25);
     }
 

From edbea73f24b8f85b05a4f6af1a8d1c7b3b63796f Mon Sep 17 00:00:00 2001
From: Tim Vernum <tim.vernum@elastic.co>
Date: Mon, 16 Jul 2018 15:43:41 +1000
Subject: [PATCH 025/260] Fix broken OpenLDAP Vagrant QA test

This was broken due to c662565 but the problem didn't get detected as
CI builds typically don't run vagrant tests
---
 .../xpack/security/authc/ldap/LdapTestUtils.java   |  2 +-
 .../java/org/elasticsearch/test/OpenLdapTests.java | 14 ++++++++++----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapTestUtils.java
index 9a9368c25e1..8bdfd02d2fc 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapTestUtils.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapTestUtils.java
@@ -62,7 +62,7 @@ public class LdapTestUtils {
 
         final SSLConfiguration sslConfiguration;
         if (useGlobalSSL) {
-            sslConfiguration = sslService.getSSLConfiguration("_global");
+            sslConfiguration = sslService.getSSLConfiguration("xpack.ssl");
         } else {
             sslConfiguration = sslService.getSSLConfiguration("xpack.security.authc.realms.foo.ssl");
         }
diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java
index c6d541b8064..f96823df019 100644
--- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java
+++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java
@@ -104,7 +104,13 @@ public class OpenLdapTests extends ESTestCase {
             builder.put("xpack.security.authc.realms." + REALM_NAME + ".ssl.truststore.path", truststore);
             mockSecureSettings.setString("xpack.security.authc.realms." + REALM_NAME + ".ssl.truststore.secure_password", "changeit");
             builder.put("xpack.security.authc.realms." + REALM_NAME + ".ssl.verification_mode", VerificationMode.CERTIFICATE);
+
+            // If not using global ssl, need to set the truststore for the "full verification" realm
+            builder.put("xpack.security.authc.realms.vmode_full.ssl.truststore.path", truststore);
+            mockSecureSettings.setString("xpack.security.authc.realms.vmode_full.ssl.truststore.secure_password", "changeit");
         }
+        builder.put("xpack.security.authc.realms.vmode_full.ssl.verification_mode", VerificationMode.FULL);
+
         globalSettings = builder.setSecureSettings(mockSecureSettings).build();
         Environment environment = TestEnvironment.newEnvironment(globalSettings);
         sslService = new SSLService(globalSettings, environment);
@@ -188,10 +194,10 @@ public class OpenLdapTests extends ESTestCase {
         Settings settings = Settings.builder()
             // The certificate used in the vagrant box is valid for "localhost", but not for "127.0.0.1"
             .put(buildLdapSettings(OPEN_LDAP_IP_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL))
-            .put("ssl.verification_mode", VerificationMode.FULL)
             .build();
 
-        RealmConfig config = new RealmConfig("oldap-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
+        // Pick up the "full" verification mode config
+        RealmConfig config = new RealmConfig("vmode_full", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
             new ThreadContext(Settings.EMPTY));
         LdapSessionFactory sessionFactory = new LdapSessionFactory(config, sslService, threadPool);
 
@@ -211,10 +217,10 @@ public class OpenLdapTests extends ESTestCase {
         Settings settings = Settings.builder()
             // The certificate used in the vagrant box is valid for "localhost" (but not for "127.0.0.1")
             .put(buildLdapSettings(OPEN_LDAP_DNS_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL))
-            .put("ssl.verification_mode", VerificationMode.FULL)
             .build();
 
-        RealmConfig config = new RealmConfig("oldap-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
+        // Pick up the "full" verification mode config
+        RealmConfig config = new RealmConfig("vmode_full", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
             new ThreadContext(Settings.EMPTY));
         LdapSessionFactory sessionFactory = new LdapSessionFactory(config, sslService, threadPool);
 

From 016e8760f0b44e816bae541a822d5d2fbb8f3021 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@gmail.com>
Date: Mon, 16 Jul 2018 10:40:36 +0200
Subject: [PATCH 026/260] Turn off real-mem breaker in single node tests

With this commit we disable the real-memory circuit breaker in tests
that inherit from `ESSingleNodeTestCase`. As this breaker is based on
real memory usage over which we have no (full) control in tests and
their purpose is also not to test the circuit breaker, we use the
deterministic circuit breaker implementation that only accounts for
explicitly reserved memory.

Closes #32047
Relates #32071
---
 .../java/org/elasticsearch/test/ESSingleNodeTestCase.java     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index a1b8f44a923..9633f56dea9 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -42,6 +42,7 @@ import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
@@ -184,6 +185,9 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b")
+            // turning on the real memory circuit breaker leads to spurious test failures. As have no full control over heap usage, we
+            // turn it off for these tests.
+            .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), false)
             .put(nodeSettings()) // allow test cases to provide their own settings or override these
             .build();
         Collection<Class<? extends Plugin>> plugins = getPlugins();

From e0cfa1689c17a660e0e3e957262e642ac760a7d8 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@gmail.com>
Date: Mon, 16 Jul 2018 10:44:04 +0200
Subject: [PATCH 027/260] Turn off real-mem breaker in REST tests

With this commit we disable the real-memory circuit breaker in REST
tests as this breaker is based on real memory usage over which we have
no (full) control in tests and the REST client is not yet ready to retry
on circuit breaker exceptions.

This is only meant as a temporary measure to avoid spurious test
failures while we ensure that the REST client can handle those
situations appropriately.

Closes #32050
Relates #31767
Relates #31986
Relates #32074
---
 .../elasticsearch/gradle/test/ClusterFormationTasks.groovy  | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index be0fb3a07c6..0349130076c 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -331,6 +331,12 @@ class ClusterFormationTasks {
         }
         // increase script compilation limit since tests can rapid-fire script compilations
         esConfig['script.max_compilations_rate'] = '2048/1m'
+        // Temporarily disable the real memory usage circuit breaker. It depends on real memory usage which we have no full control
+        // over and the REST client will not retry on circuit breaking exceptions yet (see #31986 for details). Once the REST client
+        // can retry on circuit breaking exceptions, we can revert again to the default configuration.
+        if (node.nodeVersion.major >= 7) {
+            esConfig['indices.breaker.total.use_real_memory'] = false
+        }
         esConfig.putAll(node.config.settings)
 
         Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)

From a14db2f9d6df79d5b38befc2657a689ce6d90cbf Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jimczi@apache.org>
Date: Mon, 16 Jul 2018 10:53:51 +0200
Subject: [PATCH 028/260] [Test] Mute MlJobIT#testDeleteJobAfterMissingAliases

Relates #32034
---
 .../java/org/elasticsearch/xpack/ml/integration/MlJobIT.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
index 6713e66692d..7820cbc06f5 100644
--- a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
+++ b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
@@ -438,6 +438,7 @@ public class MlJobIT extends ESRestTestCase {
                 client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32034")
     public void testDeleteJobAfterMissingAliases() throws Exception {
         String jobId = "delete-job-after-missing-alias-job";
         String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);

From ca4c4f736ae1d73d503e3e5b530e025117d15e94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Mon, 16 Jul 2018 10:54:23 +0200
Subject: [PATCH 029/260] Remove unused params from SSource and Walker (#31935)

The "source" field in SSource seems unused. If removed, it can also be removed
from the ctor, which in turn makes is possible to delete the sourceText in the
Walker class.
---
 .../org/elasticsearch/painless/antlr/Walker.java    |  6 ++----
 .../org/elasticsearch/painless/node/SSource.java    | 13 +++++--------
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index e2742ffb993..6c8d3a62e06 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -29,7 +29,6 @@ import org.antlr.v4.runtime.Recognizer;
 import org.antlr.v4.runtime.atn.PredictionMode;
 import org.antlr.v4.runtime.tree.TerminalNode;
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
@@ -107,6 +106,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.TrueContext;
 import org.elasticsearch.painless.antlr.PainlessParser.TryContext;
 import org.elasticsearch.painless.antlr.PainlessParser.VariableContext;
 import org.elasticsearch.painless.antlr.PainlessParser.WhileContext;
+import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.node.AExpression;
 import org.elasticsearch.painless.node.ANode;
 import org.elasticsearch.painless.node.AStatement;
@@ -184,7 +184,6 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
     private final CompilerSettings settings;
     private final Printer debugStream;
     private final String sourceName;
-    private final String sourceText;
     private final PainlessLookup painlessLookup;
 
     private final Deque<Reserved> reserved = new ArrayDeque<>();
@@ -198,7 +197,6 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         this.debugStream = debugStream;
         this.settings = settings;
         this.sourceName = Location.computeSourceName(sourceName);
-        this.sourceText = sourceText;
         this.globals = new Globals(new BitSet(sourceText.length()));
         this.painlessLookup = painlessLookup;
         this.source = (SSource)visit(buildAntlrTree(sourceText));
@@ -267,7 +265,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             statements.add((AStatement)visit(ctx.dstatement()));
         }
 
-        return new SSource(scriptClassInfo, settings, sourceName, sourceText, debugStream, (MainMethodReserved)reserved.pop(),
+        return new SSource(scriptClassInfo, settings, sourceName, debugStream, (MainMethodReserved)reserved.pop(),
                            location(ctx), functions, globals, statements);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 4781457a57d..cd473e2c84e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -21,9 +21,6 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Constant;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
@@ -32,6 +29,9 @@ import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.SimpleChecksAdapter;
 import org.elasticsearch.painless.WriterConstants;
+import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.node.SFunction.FunctionReserved;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
@@ -130,7 +130,6 @@ public final class SSource extends AStatement {
     private final ScriptClassInfo scriptClassInfo;
     private final CompilerSettings settings;
     private final String name;
-    private final String source;
     private final Printer debugStream;
     private final MainMethodReserved reserved;
     private final List<SFunction> functions;
@@ -141,14 +140,12 @@ public final class SSource extends AStatement {
     private final List<org.objectweb.asm.commons.Method> getMethods;
     private byte[] bytes;
 
-    public SSource(ScriptClassInfo scriptClassInfo, CompilerSettings settings, String name, String source, Printer debugStream,
-                   MainMethodReserved reserved, Location location,
-                   List<SFunction> functions, Globals globals, List<AStatement> statements) {
+    public SSource(ScriptClassInfo scriptClassInfo, CompilerSettings settings, String name, Printer debugStream,
+            MainMethodReserved reserved, Location location, List<SFunction> functions, Globals globals, List<AStatement> statements) {
         super(location);
         this.scriptClassInfo = Objects.requireNonNull(scriptClassInfo);
         this.settings = Objects.requireNonNull(settings);
         this.name = Objects.requireNonNull(name);
-        this.source = Objects.requireNonNull(source);
         this.debugStream = debugStream;
         this.reserved = Objects.requireNonNull(reserved);
         // process any synthetic functions generated by walker (because right now, thats still easy)

From 3587d8872e9d2b43c6b66f7d6ec7aafe6a7c1df0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Mon, 16 Jul 2018 11:22:42 +0200
Subject: [PATCH 030/260] [Tests] Fix failure due to changes exception message
 (#32036)

Java 11 seems to get more verbose on the ClassCastException we check for in
SearchDocumentationIT. This changes the test from asserting the exact exception
message to only checking the two classes involved are part of the message.

Closes #32029
---
 .../client/documentation/SearchDocumentationIT.java         | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
index 6d00e5d8d03..26bb4682fd9 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
@@ -295,7 +295,6 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
     }
 
     @SuppressWarnings({ "unused" })
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32029")
     public void testSearchRequestAggregations() throws IOException {
         RestHighLevelClient client = highLevelClient();
         {
@@ -338,8 +337,9 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
                     Range range = aggregations.get("by_company"); // <1>
                     // end::search-request-aggregations-get-wrongCast
                 } catch (ClassCastException ex) {
-                    assertEquals("org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms"
-                            + " cannot be cast to org.elasticsearch.search.aggregations.bucket.range.Range", ex.getMessage());
+                    String message = ex.getMessage();
+                    assertThat(message, containsString("org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms"));
+                    assertThat(message, containsString("org.elasticsearch.search.aggregations.bucket.range.Range"));
                 }
                 assertEquals(3, elasticBucket.getDocCount());
                 assertEquals(30, avg, 0.0);

From fa59bb10999b20d487ff12a890ee8cfa0f414826 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jimczi@apache.org>
Date: Mon, 16 Jul 2018 11:59:59 +0200
Subject: [PATCH 031/260] Fix BWC check after backport

Relates #31808
---
 .../java/org/elasticsearch/index/query/InnerHitBuilder.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 6bdc55d31cd..8b2db374c8d 100644
--- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -199,7 +199,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
             boolean hasChildren = in.readBoolean();
             assert hasChildren == false;
         }
-        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
             this.innerCollapseBuilder = in.readOptionalWriteable(CollapseBuilder::new);
         }
     }
@@ -247,7 +247,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
             }
         }
         out.writeOptionalWriteable(highlightBuilder);
-        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+        if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
             out.writeOptionalWriteable(innerCollapseBuilder);
         }
     }

From 44f0c1df39ebc34e9613e3e5bcad78a3308a9068 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jimczi@apache.org>
Date: Mon, 16 Jul 2018 12:03:28 +0200
Subject: [PATCH 032/260] Unmute field collapsing rest tests

BWC tests can run now that master and 6x branch are aligned.
Closes #32055
---
 .../test/search.inner_hits/10_basic.yml         |  9 ++++-----
 .../test/search/110_field_collapsing.yml        | 17 +++--------------
 2 files changed, 7 insertions(+), 19 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml
index 8f162ae2eb2..884a50507c7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml
@@ -13,9 +13,8 @@ setup:
 ---
 "Nested inner hits":
     - skip:
-        version: "all"
-        reason: "https://github.com/elastic/elasticsearch/issues/32055"
-
+        version: " - 6.1.99"
+        reason: "<= 6.1 nodes don't always include index or id in nested inner hits"
     - do:
         index:
           index: test
@@ -46,8 +45,8 @@ setup:
 "Nested doc version and seqIDs":
 
     - skip:
-        version: "all"
-        reason: "https://github.com/elastic/elasticsearch/issues/32055"
+        version: " - 6.3.99"
+        reason:  "object notation for docvalue_fields was introduced in 6.4"
 
     - do:
         index:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
index 39597b1fbbe..2dfd868d66b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
@@ -107,9 +107,6 @@ setup:
 
 ---
 "field collapsing and inner_hits":
-  - skip:
-      version: "all"
-      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:
@@ -149,9 +146,6 @@ setup:
 
 ---
 "field collapsing, inner_hits and maxConcurrentGroupRequests":
-  - skip:
-      version: "all"
-      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:
@@ -232,9 +226,6 @@ setup:
 
 ---
 "no hits and inner_hits":
-  - skip:
-      version: "all"
-      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:
@@ -249,9 +240,6 @@ setup:
 
 ---
 "field collapsing and multiple inner_hits":
-  - skip:
-      version: "all"
-      reason: "https://github.com/elastic/elasticsearch/issues/32055"
 
   - do:
       search:
@@ -304,9 +292,10 @@ setup:
 
 ---
 "field collapsing, inner_hits and version":
+
   - skip:
-      version: "all"
-      reason: "https://github.com/elastic/elasticsearch/issues/32055"
+      version: " - 6.1.0"
+      reason:  "bug fixed in 6.1.1"
 
   - do:
       search:

From 1fef139c11065aee746a18f4d76ca65eb369ece3 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel@mitterdorfer.name>
Date: Mon, 16 Jul 2018 13:50:17 +0200
Subject: [PATCH 033/260] Ensure only parent breaker trips in unit test

With this commit we raise the limit of the child circuit breaker used in
the unit test for the circuit breaker service so it is high enough to trip
only the parent circuit breaker. The previous limit was 300 bytes but
theoretically (considering overhead) we could reach 346 bytes. Thus any
value larger than 300 bytes could trip the child circuit breaker leading
to spurious failures.

Relates #31767
---
 .../indices/breaker/HierarchyCircuitBreakerServiceTests.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
index 00bd15d244f..a73cf8630fe 100644
--- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
@@ -206,7 +206,7 @@ public class HierarchyCircuitBreakerServiceTests extends ESTestCase {
         Settings clusterSettings = Settings.builder()
             .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), Boolean.TRUE)
             .put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "200b")
-            .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "300b")
+            .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "350b")
             .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 2)
             .build();
 

From a3b608d616fe8f9b8bb10bb60a97a9f39ec53a6c Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Mon, 16 Jul 2018 15:25:45 +0200
Subject: [PATCH 034/260] [Rollup] Fix duplicate field names in test (#32075)

This commit ensures that random field names do not clash with the explicit field names
set by the tests.

Closes #32067
---
 .../elasticsearch/xpack/core/rollup/ConfigTestHelpers.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
index 3d82ac118f5..3e4e4a84d2f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
@@ -94,7 +94,7 @@ public class ConfigTestHelpers {
         if (ESTestCase.randomBoolean()) {
             dateHistoBuilder.setDelay(new DateHistogramInterval(randomPositiveTimeValue()));
         }
-        dateHistoBuilder.setField(ESTestCase.randomAlphaOfLengthBetween(1, 10 ));
+        dateHistoBuilder.setField(ESTestCase.randomAlphaOfLengthBetween(5, 10));
         return dateHistoBuilder;
     }
 
@@ -112,8 +112,8 @@ public class ConfigTestHelpers {
     }
 
     public static  List<String> getFields() {
-        return IntStream.range(0, ESTestCase.randomIntBetween(1,10))
-                .mapToObj(n -> ESTestCase.randomAlphaOfLengthBetween(1,10))
+        return IntStream.range(0, ESTestCase.randomIntBetween(1, 10))
+                .mapToObj(n -> ESTestCase.randomAlphaOfLengthBetween(5, 10))
                 .collect(Collectors.toList());
     }
 

From ef7ccd1c074b7fb9a2d1e388d517e6d20c131903 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ikakavas@protonmail.com>
Date: Mon, 16 Jul 2018 16:41:56 +0300
Subject: [PATCH 035/260] [TEST] Consistent algorithm usage (#32077)

Ensure that the same algorithm is used for settings and
change password requests for consistency, even if we
do not expext to reach the code where the algorithm is
checked for now.
Completes a7eaa409e804f218aa06fd02d9166b9a5998b48a
---
 .../TransportChangePasswordActionTests.java     | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
index 516b33cbacc..aabaa40381f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
@@ -49,18 +49,16 @@ public class TransportChangePasswordActionTests extends ESTestCase {
 
     public void testAnonymousUser() {
         final String hashingAlgorithm = randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9");
-        Settings settings = Settings.builder().put(AnonymousUser.ROLES_SETTING.getKey(), "superuser").build();
+        Settings settings = Settings.builder().put(AnonymousUser.ROLES_SETTING.getKey(), "superuser")
+            .put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), hashingAlgorithm).build();
         AnonymousUser anonymousUser = new AnonymousUser(settings);
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
-        Settings passwordHashingSettings = Settings.builder().
-            put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), hashingAlgorithm).build();
-        TransportService transportService = new TransportService(passwordHashingSettings, mock(Transport.class), null,
+        TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet());
         TransportChangePasswordAction action = new TransportChangePasswordAction(settings, transportService,
             mock(ActionFilters.class), usersStore);
-
-        ChangePasswordRequest request = new ChangePasswordRequest();
         // Request will fail before the request hashing algorithm is checked, but we use the same algorithm as in settings for consistency
+        ChangePasswordRequest request = new ChangePasswordRequest();
         request.username(anonymousUser.principal());
         request.passwordHash(Hasher.resolve(hashingAlgorithm).hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING));
 
@@ -89,14 +87,13 @@ public class TransportChangePasswordActionTests extends ESTestCase {
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
         Settings passwordHashingSettings = Settings.builder().
             put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), hashingAlgorithm).build();
-        TransportService transportService = new TransportService(passwordHashingSettings, mock(Transport.class), null,
+        TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet());
-        TransportChangePasswordAction action = new TransportChangePasswordAction(Settings.EMPTY, transportService,
+        TransportChangePasswordAction action = new TransportChangePasswordAction(passwordHashingSettings, transportService,
             mock(ActionFilters.class), usersStore);
-
+        // Request will fail before the request hashing algorithm is checked, but we use the same algorithm as in settings for consistency
         ChangePasswordRequest request = new ChangePasswordRequest();
         request.username(randomFrom(SystemUser.INSTANCE.principal(), XPackUser.INSTANCE.principal()));
-        // Request will fail before the request hashing algorithm is checked, but we use the same algorithm as in settings for consistency
         request.passwordHash(Hasher.resolve(hashingAlgorithm).hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING));
 
         final AtomicReference<Throwable> throwableRef = new AtomicReference<>();

From 59191b4998afb7dd7093e44d14f06f4f3e7c4740 Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Mon, 16 Jul 2018 10:47:46 -0400
Subject: [PATCH 036/260] [Rollup] Replace RollupIT with a ESRestTestCase
 version (#31977)

The old RollupIT was a node IT, an flaky for a number of reasons.
This new version is an ESRestTestCase and should be a little more robust.

This was added to the multi-node QA tests as that seemed like the most
appropriate location.  It didn't seem necessary to create a whole new
QA module.

Note: The only test that was ported was the "Big" test for validating
a larger dataset.  The rest of the tests are represented in existing
yaml tests.

Closes #31258
Closes #30232
Related to #30290
---
 x-pack/plugin/rollup/build.gradle             |  28 -
 .../elasticsearch/xpack/rollup/RollupIT.java  | 498 ------------------
 .../elasticsearch/multi_node/RollupIT.java    | 326 ++++++++++++
 3 files changed, 326 insertions(+), 526 deletions(-)
 delete mode 100644 x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupIT.java
 create mode 100644 x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java

diff --git a/x-pack/plugin/rollup/build.gradle b/x-pack/plugin/rollup/build.gradle
index 18ef7abee5c..ff9c30ed9a9 100644
--- a/x-pack/plugin/rollup/build.gradle
+++ b/x-pack/plugin/rollup/build.gradle
@@ -1,6 +1,3 @@
-import com.carrotsearch.gradle.junit4.RandomizedTestingTask
-import org.elasticsearch.gradle.BuildPlugin
-
 evaluationDependsOn(xpackModule('core'))
 
 apply plugin: 'elasticsearch.esplugin'
@@ -23,33 +20,8 @@ dependencies {
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-dependencyLicenses {
-    ignoreSha 'x-pack-core'
-}
-
 run {
     plugin xpackModule('core')
 }
 
 integTest.enabled = false
-
-
-// Instead we create a separate task to run the
-// tests based on ESIntegTestCase
-task internalClusterTest(type: RandomizedTestingTask,
-        group: JavaBasePlugin.VERIFICATION_GROUP,
-        description: 'Multi-node tests',
-        dependsOn: test.dependsOn) {
-    configure(BuildPlugin.commonTestConfig(project))
-    classpath = project.test.classpath
-    testClassesDirs = project.test.testClassesDirs
-    include '**/*IT.class'
-    systemProperty 'es.set.netty.runtime.available.processors', 'false'
-}
-check.dependsOn internalClusterTest
-internalClusterTest.mustRunAfter test
-
-// also add an "alias" task to make typing on the command line easier task icTest {
-task icTest {
-    dependsOn internalClusterTest
-}
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupIT.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupIT.java
deleted file mode 100644
index 157cd6a5b9d..00000000000
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupIT.java
+++ /dev/null
@@ -1,498 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.rollup;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.license.LicenseService;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.search.aggregations.Aggregation;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.transport.Netty4Plugin;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.rollup.action.DeleteRollupJobAction;
-import org.elasticsearch.xpack.core.rollup.action.GetRollupJobsAction;
-import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction;
-import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
-import org.elasticsearch.xpack.core.rollup.action.StartRollupJobAction;
-import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction;
-import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
-import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
-import org.elasticsearch.xpack.core.rollup.job.IndexerState;
-import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
-import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
-import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus;
-import org.hamcrest.Matchers;
-import org.joda.time.DateTime;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram;
-import static org.hamcrest.core.IsEqual.equalTo;
-
-@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
-public class RollupIT extends ESIntegTestCase {
-
-    private String taskId = "test-bigID";
-
-    @Override
-    protected boolean ignoreExternalCluster() {
-        return true;
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LocalStateRollup.class, CommonAnalysisPlugin.class, Netty4Plugin.class);
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return nodePlugins();
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder builder = Settings.builder();
-        builder.put(XPackSettings.ROLLUP_ENABLED.getKey(), true);
-        builder.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
-        builder.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
-        return builder.build();
-    }
-
-    @Override
-    protected Settings externalClusterClientSettings() {
-        return nodeSettings(0);
-    }
-
-    @Override
-    protected Settings transportClientSettings() {
-        return Settings.builder().put(super.transportClientSettings())
-                .put(XPackSettings.ROLLUP_ENABLED.getKey(), true)
-                .put(XPackSettings.SECURITY_ENABLED.getKey(), false)
-                .build();
-    }
-
-    @Before
-    public void createIndex() {
-        client().admin().indices().prepareCreate("test-1").addMapping("doc", "{\"doc\": {\"properties\": {" +
-                "\"date_histo\": {\"type\": \"date\"}, " +
-                "\"histo\": {\"type\": \"integer\"}, " +
-                "\"terms\": {\"type\": \"keyword\"}}}}", XContentType.JSON).get();
-        client().admin().cluster().prepareHealth("test-1").setWaitForYellowStatus().get();
-
-        BulkRequestBuilder bulk = client().prepareBulk();
-        Map<String, Object> source = new HashMap<>(3);
-        for (int i = 0; i < 20; i++) {
-            for (int j = 0; j < 20; j++) {
-                for (int k = 0; k < 20; k++) {
-                    source.put("date_histo", new DateTime().minusDays(i).toString());
-                    source.put("histo", Integer.toString(j * 100));
-                    source.put("terms", Integer.toString(k * 100));
-                    source.put("foo", k);
-                    bulk.add(new IndexRequest("test-1", "doc").source(source));
-                    source.clear();
-                }
-            }
-        }
-        bulk.get();
-        client().admin().indices().prepareRefresh("test-1").get();
-    }
-
-    public void testGetJob() throws ExecutionException, InterruptedException {
-        MetricConfig metricConfig = new MetricConfig.Builder()
-                .setField("foo")
-                .setMetrics(Arrays.asList("sum", "min", "max", "avg"))
-                .build();
-
-        DateHistoGroupConfig.Builder datehistoGroupConfig = new DateHistoGroupConfig.Builder();
-        datehistoGroupConfig.setField("date_histo");
-        datehistoGroupConfig.setInterval(new DateHistogramInterval("1d"));
-
-        GroupConfig.Builder groupConfig = new GroupConfig.Builder();
-        groupConfig.setDateHisto(datehistoGroupConfig.build());
-
-
-        RollupJobConfig.Builder config = new RollupJobConfig.Builder();
-        config.setIndexPattern("test-1");
-        config.setRollupIndex("rolled");
-        config.setId("testGet");
-        config.setGroupConfig(groupConfig.build());
-        config.setMetricsConfig(Collections.singletonList(metricConfig));
-        config.setCron("* * * * * ? *");
-        config.setPageSize(10);
-
-        PutRollupJobAction.Request request = new PutRollupJobAction.Request();
-        request.setConfig(config.build());
-        client().execute(PutRollupJobAction.INSTANCE, request).get();
-
-        GetRollupJobsAction.Request getRequest = new GetRollupJobsAction.Request("testGet");
-        GetRollupJobsAction.Response response = client().execute(GetRollupJobsAction.INSTANCE, getRequest).get();
-        assertThat(response.getJobs().size(), equalTo(1));
-        assertThat(response.getJobs().get(0).getJob().getId(), equalTo("testGet"));
-    }
-
-    public void testIndexPattern() throws Exception {
-        MetricConfig metricConfig = new MetricConfig.Builder()
-                .setField("foo")
-                .setMetrics(Arrays.asList("sum", "min", "max", "avg"))
-                .build();
-
-        DateHistoGroupConfig.Builder datehistoGroupConfig = new DateHistoGroupConfig.Builder();
-        datehistoGroupConfig.setField("date_histo");
-        datehistoGroupConfig.setInterval(new DateHistogramInterval("1d"));
-
-        GroupConfig.Builder groupConfig = new GroupConfig.Builder();
-        groupConfig.setDateHisto(datehistoGroupConfig.build());
-
-
-        RollupJobConfig.Builder config = new RollupJobConfig.Builder();
-        config.setIndexPattern("test-*");
-        config.setId("testIndexPattern");
-        config.setRollupIndex("rolled");
-        config.setGroupConfig(groupConfig.build());
-        config.setMetricsConfig(Collections.singletonList(metricConfig));
-        config.setCron("* * * * * ? *");
-        config.setPageSize(10);
-
-        PutRollupJobAction.Request request = new PutRollupJobAction.Request();
-        request.setConfig(config.build());
-        client().execute(PutRollupJobAction.INSTANCE, request).get();
-
-        StartRollupJobAction.Request startRequest = new StartRollupJobAction.Request("testIndexPattern");
-        StartRollupJobAction.Response startResponse = client().execute(StartRollupJobAction.INSTANCE, startRequest).get();
-        Assert.assertThat(startResponse.isStarted(), equalTo(true));
-
-        // Make sure it started
-        ESTestCase.assertBusy(() -> {
-            RollupJobStatus rollupJobStatus = getRollupJobStatus("testIndexPattern");
-            if (rollupJobStatus == null) {
-                fail("null");
-            }
-
-            IndexerState state = rollupJobStatus.getIndexerState();
-            assertTrue(state.equals(IndexerState.STARTED) || state.equals(IndexerState.INDEXING));
-        }, 60, TimeUnit.SECONDS);
-
-        // And wait for it to finish
-        ESTestCase.assertBusy(() -> {
-            RollupJobStatus rollupJobStatus = getRollupJobStatus("testIndexPattern");
-            if (rollupJobStatus == null) {
-                fail("null");
-            }
-
-            IndexerState state = rollupJobStatus.getIndexerState();
-            assertTrue(state.equals(IndexerState.STARTED) && rollupJobStatus.getPosition() != null);
-        }, 60, TimeUnit.SECONDS);
-
-        GetRollupJobsAction.Request getRequest = new GetRollupJobsAction.Request("testIndexPattern");
-        GetRollupJobsAction.Response response = client().execute(GetRollupJobsAction.INSTANCE, getRequest).get();
-        Assert.assertThat(response.getJobs().size(), equalTo(1));
-        Assert.assertThat(response.getJobs().get(0).getJob().getId(), equalTo("testIndexPattern"));
-
-        GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().addIndices("rolled").get();
-        Assert.assertThat(getIndexResponse.indices().length, Matchers.greaterThan(0));
-    }
-
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30290")
-    public void testTwoJobsStartStopDeleteOne() throws Exception {
-        MetricConfig metricConfig = new MetricConfig.Builder()
-                .setField("foo")
-                .setMetrics(Arrays.asList("sum", "min", "max", "avg"))
-                .build();
-
-        DateHistoGroupConfig.Builder datehistoGroupConfig = new DateHistoGroupConfig.Builder();
-        datehistoGroupConfig.setField("date_histo");
-        datehistoGroupConfig.setInterval(new DateHistogramInterval("1d"));
-
-        GroupConfig.Builder groupConfig = new GroupConfig.Builder();
-        groupConfig.setDateHisto(datehistoGroupConfig.build());
-
-
-        RollupJobConfig.Builder config = new RollupJobConfig.Builder();
-        config.setIndexPattern("test-1");
-        config.setRollupIndex("rolled");
-        config.setId("job1");
-        config.setGroupConfig(groupConfig.build());
-        config.setMetricsConfig(Collections.singletonList(metricConfig));
-        config.setCron("* * * * * ? *");
-        config.setPageSize(10);
-
-        PutRollupJobAction.Request request = new PutRollupJobAction.Request();
-        request.setConfig(config.build());
-        client().execute(PutRollupJobAction.INSTANCE, request).get();
-
-        RollupJobConfig.Builder config2 = new RollupJobConfig.Builder();
-        config2.setIndexPattern("test-1");
-        config2.setRollupIndex("rolled");
-        config2.setId("job2");
-        config2.setGroupConfig(groupConfig.build());
-        config2.setMetricsConfig(Collections.singletonList(metricConfig));
-        config2.setCron("* * * * * ? *");
-        config2.setPageSize(10);
-
-        PutRollupJobAction.Request request2 = new PutRollupJobAction.Request();
-        request2.setConfig(config2.build());
-        client().execute(PutRollupJobAction.INSTANCE, request2).get();
-
-        StartRollupJobAction.Request startRequest = new StartRollupJobAction.Request("job1");
-        StartRollupJobAction.Response response = client().execute(StartRollupJobAction.INSTANCE, startRequest).get();
-        Assert.assertThat(response.isStarted(), equalTo(true));
-
-        // Make sure it started
-        ESTestCase.assertBusy(() -> {
-            RollupJobStatus rollupJobStatus = getRollupJobStatus("job1");
-            if (rollupJobStatus == null) {
-                fail("null");
-            }
-
-            IndexerState state = rollupJobStatus.getIndexerState();
-            assertTrue(state.equals(IndexerState.STARTED) || state.equals(IndexerState.INDEXING));
-        }, 60, TimeUnit.SECONDS);
-
-        //but not the other task
-        ESTestCase.assertBusy(() -> {
-            RollupJobStatus rollupJobStatus = getRollupJobStatus("job2");
-
-            IndexerState state = rollupJobStatus.getIndexerState();
-            assertTrue(state.equals(IndexerState.STOPPED));
-        }, 60, TimeUnit.SECONDS);
-
-        // Delete the task
-        DeleteRollupJobAction.Request deleteRequest = new DeleteRollupJobAction.Request("job1");
-        DeleteRollupJobAction.Response deleteResponse = client().execute(DeleteRollupJobAction.INSTANCE, deleteRequest).get();
-        Assert.assertTrue(deleteResponse.isAcknowledged());
-
-        // Make sure the first job's task is gone
-        ESTestCase.assertBusy(() -> {
-            RollupJobStatus rollupJobStatus = getRollupJobStatus("job1");
-            assertTrue(rollupJobStatus == null);
-        }, 60, TimeUnit.SECONDS);
-
-        // And that we don't see it in the GetJobs API
-        GetRollupJobsAction.Request getRequest = new GetRollupJobsAction.Request("job1");
-        GetRollupJobsAction.Response getResponse = client().execute(GetRollupJobsAction.INSTANCE, getRequest).get();
-        Assert.assertThat(getResponse.getJobs().size(), equalTo(0));
-
-        // But make sure the other job is still there
-        getRequest = new GetRollupJobsAction.Request("job2");
-        getResponse = client().execute(GetRollupJobsAction.INSTANCE, getRequest).get();
-        Assert.assertThat(getResponse.getJobs().size(), equalTo(1));
-        Assert.assertThat(getResponse.getJobs().get(0).getJob().getId(), equalTo("job2"));
-
-        // and still STOPPED
-        ESTestCase.assertBusy(() -> {
-            RollupJobStatus rollupJobStatus = getRollupJobStatus("job2");
-
-            IndexerState state = rollupJobStatus.getIndexerState();
-            assertTrue(state.equals(IndexerState.STOPPED));
-        }, 60, TimeUnit.SECONDS);
-    }
-
-    public void testBig() throws Exception {
-
-        client().admin().indices().prepareCreate("test-big")
-                .addMapping("test-big", "{\"test-big\": {\"properties\": {\"timestamp\": {\"type\": \"date\"}, " +
-                    "\"thefield\": {\"type\": \"integer\"}}}}", XContentType.JSON)
-                .setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get();
-        client().admin().cluster().prepareHealth("test-big").setWaitForYellowStatus().get();
-
-        client().admin().indices().prepareCreate("test-verify")
-                .addMapping("test-big", "{\"test-big\": {\"properties\": {\"timestamp\": {\"type\": \"date\"}, " +
-                        "\"thefield\": {\"type\": \"integer\"}}}}", XContentType.JSON)
-                .setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get();
-        client().admin().cluster().prepareHealth("test-verify").setWaitForYellowStatus().get();
-
-        BulkRequestBuilder bulk = client().prepareBulk();
-        Map<String, Object> source = new HashMap<>(3);
-
-        int numDays = 90;
-        int numDocsPerDay = 100;
-
-        for (int i = 0; i < numDays; i++) {
-            DateTime ts = new DateTime().minusDays(i);
-            for (int j = 0; j < numDocsPerDay; j++) {
-
-                int value = ESTestCase.randomIntBetween(0,100);
-                source.put("timestamp", ts.toString());
-                source.put("thefield", value);
-                bulk.add(new IndexRequest("test-big", "test-big").source(source));
-                bulk.add(new IndexRequest("test-verify", "test-big").source(source));
-                source.clear();
-            }
-
-            bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
-            bulk.get();
-            bulk = client().prepareBulk();
-            logger.info("Day: [" + i + "]: " + ts.toString() + " [" + ts.getMillis() + "]" );
-        }
-
-
-        client().admin().indices().prepareRefresh("test-big").get();
-        client().admin().indices().prepareRefresh("test-verify").get();
-
-        MetricConfig metricConfig = new MetricConfig.Builder()
-                .setField("thefield")
-                .setMetrics(Arrays.asList("sum", "min", "max", "avg"))
-                .build();
-
-        DateHistoGroupConfig.Builder datehistoGroupConfig = new DateHistoGroupConfig.Builder();
-        datehistoGroupConfig.setField("timestamp");
-        datehistoGroupConfig.setInterval(new DateHistogramInterval("1d"));
-
-        GroupConfig.Builder groupConfig = new GroupConfig.Builder();
-        groupConfig.setDateHisto(datehistoGroupConfig.build());
-
-        RollupJobConfig.Builder config = new RollupJobConfig.Builder();
-        config.setIndexPattern("test-big");
-        config.setRollupIndex("rolled");
-        config.setId(taskId);
-        config.setGroupConfig(groupConfig.build());
-        config.setMetricsConfig(Collections.singletonList(metricConfig));
-        config.setCron("* * * * * ? *");
-        config.setPageSize(1000);
-
-        PutRollupJobAction.Request request = new PutRollupJobAction.Request();
-        request.setConfig(config.build());
-        client().execute(PutRollupJobAction.INSTANCE, request).get();
-
-        StartRollupJobAction.Request startRequest = new StartRollupJobAction.Request(taskId);
-        StartRollupJobAction.Response response = client().execute(StartRollupJobAction.INSTANCE, startRequest).get();
-        Assert.assertThat(response.isStarted(), equalTo(true));
-
-        ESTestCase.assertBusy(() -> {
-            RollupJobStatus rollupJobStatus = getRollupJobStatus(taskId);
-            if (rollupJobStatus == null) {
-                fail("null");
-            }
-
-            IndexerState state = rollupJobStatus.getIndexerState();
-            logger.error("state: [" + state + "]");
-            assertTrue(state.equals(IndexerState.STARTED) && rollupJobStatus.getPosition() != null);
-        }, 60, TimeUnit.SECONDS);
-
-        RollupJobStatus rollupJobStatus = getRollupJobStatus(taskId);
-        if (rollupJobStatus == null) {
-            Assert.fail("rollup job status should not be null");
-        }
-
-        client().admin().indices().prepareRefresh("rolled").get();
-
-        SearchResponse count = client().prepareSearch("rolled").setSize(10).get();
-        // total document is numDays minus 1 because we don't build rollup for
-        // buckets that are not full (bucket for the current day).
-        Assert.assertThat(count.getHits().totalHits, equalTo(Long.valueOf(numDays-1)));
-
-        if (ESTestCase.randomBoolean()) {
-            client().admin().indices().prepareDelete("test-big").get();
-            client().admin().indices().prepareRefresh().get();
-        }
-
-        // Execute the rollup search
-        SearchRequest rollupRequest = new SearchRequest("rolled")
-                .source(new SearchSourceBuilder()
-                        .aggregation(dateHistogram("timestamp")
-                                .interval(1000*86400)
-                                .field("timestamp"))
-                        .size(0));
-        SearchResponse searchResponse = client().execute(RollupSearchAction.INSTANCE, rollupRequest).get();
-        Assert.assertNotNull(searchResponse);
-
-        // And a regular search against the verification index
-        SearchRequest verifyRequest = new SearchRequest("test-verify")
-                .source(new SearchSourceBuilder()
-                        .aggregation(dateHistogram("timestamp")
-                                .interval(1000*86400)
-                                .field("timestamp"))
-                        .size(0));
-        SearchResponse verifyResponse = client().execute(SearchAction.INSTANCE, verifyRequest).get();
-
-        Map<String, Aggregation> rollupAggs = searchResponse.getAggregations().asMap();
-
-        for (Aggregation agg : verifyResponse.getAggregations().asList()) {
-            Aggregation rollupAgg = rollupAggs.get(agg.getName());
-
-            Assert.assertNotNull(rollupAgg);
-            Assert.assertThat(rollupAgg.getType(), equalTo(agg.getType()));
-            verifyAgg((InternalDateHistogram)agg, (InternalDateHistogram)rollupAgg);
-        }
-
-        // And a quick sanity check for doc type
-        SearchRequest rollupRawRequest = new SearchRequest("rolled")
-                .source(new SearchSourceBuilder().query(new MatchAllQueryBuilder())
-                        .size(1));
-        SearchResponse searchRawResponse = client().execute(SearchAction.INSTANCE, rollupRawRequest).get();
-        Assert.assertNotNull(searchRawResponse);
-        assertThat(searchRawResponse.getHits().getAt(0).getType(), equalTo("_doc"));
-    }
-
-    private void verifyAgg(InternalDateHistogram verify, InternalDateHistogram rollup) {
-        for (int i = 0; i < rollup.getBuckets().size(); i++) {
-            InternalDateHistogram.Bucket verifyBucket = verify.getBuckets().get(i);
-            InternalDateHistogram.Bucket rollupBucket = rollup.getBuckets().get(i);
-            Assert.assertThat(rollupBucket.getDocCount(), equalTo(verifyBucket.getDocCount()));
-            Assert.assertThat(((DateTime)rollupBucket.getKey()).getMillis(), equalTo(((DateTime)verifyBucket.getKey()).getMillis()));
-            Assert.assertTrue(rollupBucket.getAggregations().equals(verifyBucket.getAggregations()));
-        }
-    }
-
-    private RollupJobStatus getRollupJobStatus(final String taskId) {
-        final GetRollupJobsAction.Request request = new GetRollupJobsAction.Request(taskId);
-        final GetRollupJobsAction.Response response = client().execute(GetRollupJobsAction.INSTANCE, request).actionGet();
-
-        if (response.getJobs() != null && response.getJobs().isEmpty() == false) {
-            assertThat("Expect 1 rollup job with id " + taskId, response.getJobs().size(), equalTo(1));
-            return response.getJobs().iterator().next().getStatus();
-        }
-        return null;
-    }
-
-    @After
-    public void cleanup() throws ExecutionException, InterruptedException {
-        GetRollupJobsAction.Request getRequest = new GetRollupJobsAction.Request("_all");
-        GetRollupJobsAction.Response response = client().execute(GetRollupJobsAction.INSTANCE, getRequest).get();
-
-        for (GetRollupJobsAction.JobWrapper job : response.getJobs()) {
-            StopRollupJobAction.Request stopRequest = new StopRollupJobAction.Request(job.getJob().getId());
-            try {
-                client().execute(StopRollupJobAction.INSTANCE, stopRequest).get();
-            } catch (ElasticsearchException e) {
-                //
-            }
-
-            DeleteRollupJobAction.Request deleteRequest = new DeleteRollupJobAction.Request(job.getJob().getId());
-            client().execute(DeleteRollupJobAction.INSTANCE, deleteRequest).get();
-        }
-    }
-}
diff --git a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
new file mode 100644
index 00000000000..b0142ae1418
--- /dev/null
+++ b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
@@ -0,0 +1,326 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.multi_node;
+
+import org.apache.http.HttpStatus;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.core.rollup.job.RollupJob;
+import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.junit.After;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.isOneOf;
+
+public class RollupIT extends ESRestTestCase {
+
+    @Override
+    protected Settings restClientSettings() {
+        return getClientSettings("super-user", "x-pack-super-password");
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        return getClientSettings("super-user", "x-pack-super-password");
+    }
+
+    private Settings getClientSettings(final String username, final String password) {
+        final String token = basicAuthHeaderValue(username, new SecureString(password.toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    static Map<String, Object> toMap(Response response) throws IOException {
+        return toMap(EntityUtils.toString(response.getEntity()));
+    }
+
+    static Map<String, Object> toMap(String response) throws IOException {
+        return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false);
+    }
+
+    @After
+    public void clearRollupMetadata() throws Exception {
+        deleteAllJobs();
+        waitForPendingTasks();
+        // indices will be deleted by the ESRestTestCase class
+    }
+
+    public void testBigRollup() throws Exception {
+        final int numDocs = 200;
+
+        // index documents for the rollup job
+        final StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < numDocs; i++) {
+            bulk.append("{\"index\":{\"_index\":\"rollup-docs\",\"_type\":\"_doc\"}}\n");
+            ZonedDateTime zdt = ZonedDateTime.ofInstant(Instant.ofEpochSecond(1531221196 + (60*i)), ZoneId.of("UTC"));
+            String date = zdt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
+            bulk.append("{\"timestamp\":\"").append(date).append("\",\"value\":").append(i).append("}\n");
+        }
+        bulk.append("\r\n");
+
+        final Request bulkRequest = new Request("POST", "/_bulk");
+        bulkRequest.addParameter("refresh", "true");
+        bulkRequest.setJsonEntity(bulk.toString());
+        client().performRequest(bulkRequest);
+        // create the rollup job
+        final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-job-test");
+        createRollupJobRequest.setJsonEntity("{"
+            + "\"index_pattern\":\"rollup-*\","
+            + "\"rollup_index\":\"results-rollup\","
+            + "\"cron\":\"*/1 * * * * ?\","             // fast cron and big page size so test runs quickly
+            + "\"page_size\":20,"
+            + "\"groups\":{"
+            + "    \"date_histogram\":{"
+            + "        \"field\":\"timestamp\","
+            + "        \"interval\":\"5m\""
+            + "      }"
+            + "},"
+            + "\"metrics\":["
+            + "    {\"field\":\"value\",\"metrics\":[\"min\",\"max\",\"sum\"]}"
+            + "]"
+            + "}");
+
+        Map<String, Object> createRollupJobResponse = toMap(client().performRequest(createRollupJobRequest));
+        assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+        // start the rollup job
+        final Request startRollupJobRequest = new Request("POST", "_xpack/rollup/job/rollup-job-test/_start");
+        Map<String, Object> startRollupJobResponse = toMap(client().performRequest(startRollupJobRequest));
+        assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
+
+        assertRollUpJob("rollup-job-test");
+
+        // Wait for the job to finish, by watching how many rollup docs we've indexed
+        assertBusy(() -> {
+            final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/rollup-job-test");
+            Response getRollupJobResponse = client().performRequest(getRollupJobRequest);
+            assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+
+            Map<String, Object> job = getJob(getRollupJobResponse, "rollup-job-test");
+            if (job != null) {
+                assertThat(ObjectPath.eval("status.job_state", job), equalTo("started"));
+                assertThat(ObjectPath.eval("stats.rollups_indexed", job), equalTo(41));
+            }
+        }, 30L, TimeUnit.SECONDS);
+
+        // Refresh the rollup index to make sure all newly indexed docs are searchable
+        final Request refreshRollupIndex = new Request("POST", "results-rollup/_refresh");
+        toMap(client().performRequest(refreshRollupIndex));
+
+        String jsonRequestBody = "{\n" +
+            "  \"size\": 0,\n" +
+            "  \"query\": {\n" +
+            "    \"match_all\": {}\n" +
+            "  },\n" +
+            "  \"aggs\": {\n" +
+            "    \"date_histo\": {\n" +
+            "      \"date_histogram\": {\n" +
+            "        \"field\": \"timestamp\",\n" +
+            "        \"interval\": \"1h\"\n" +
+            "      },\n" +
+            "      \"aggs\": {\n" +
+            "        \"the_max\": {\n" +
+            "          \"max\": {\n" +
+            "            \"field\": \"value\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  }\n" +
+            "}";
+
+        Request request = new Request("GET", "rollup-docs/_search");
+        request.setJsonEntity(jsonRequestBody);
+        Response liveResponse = client().performRequest(request);
+        Map<String, Object> liveBody = toMap(liveResponse);
+
+        request = new Request("GET", "results-rollup/_rollup_search");
+        request.setJsonEntity(jsonRequestBody);
+        Response rollupResponse = client().performRequest(request);
+        Map<String, Object> rollupBody = toMap(rollupResponse);
+
+        // Do the live agg results match the rollup agg results?
+        assertThat(ObjectPath.eval("aggregations.date_histo.buckets", liveBody),
+            equalTo(ObjectPath.eval("aggregations.date_histo.buckets", rollupBody)));
+
+        request = new Request("GET", "rollup-docs/_rollup_search");
+        request.setJsonEntity(jsonRequestBody);
+        Response liveRollupResponse = client().performRequest(request);
+        Map<String, Object> liveRollupBody = toMap(liveRollupResponse);
+
+        // Does searching the live index via rollup_search work match the live search?
+        assertThat(ObjectPath.eval("aggregations.date_histo.buckets", liveBody),
+            equalTo(ObjectPath.eval("aggregations.date_histo.buckets", liveRollupBody)));
+
+    }
+
+    @SuppressWarnings("unchecked")
+    private void assertRollUpJob(final String rollupJob) throws Exception {
+        String[] states = new String[]{"indexing", "started"};
+        waitForRollUpJob(rollupJob, states);
+
+        // check that the rollup job is started using the RollUp API
+        final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/" + rollupJob);
+        Map<String, Object> getRollupJobResponse = toMap(client().performRequest(getRollupJobRequest));
+        Map<String, Object> job = getJob(getRollupJobResponse, rollupJob);
+        if (job != null) {
+            assertThat(ObjectPath.eval("status.job_state", job), isOneOf(states));
+        }
+
+        // check that the rollup job is started using the Tasks API
+        final Request taskRequest = new Request("GET", "_tasks");
+        taskRequest.addParameter("detailed", "true");
+        taskRequest.addParameter("actions", "xpack/rollup/*");
+        Map<String, Object> taskResponse = toMap(client().performRequest(taskRequest));
+        Map<String, Object> taskResponseNodes = (Map<String, Object>) taskResponse.get("nodes");
+        Map<String, Object> taskResponseNode = (Map<String, Object>) taskResponseNodes.values().iterator().next();
+        Map<String, Object> taskResponseTasks = (Map<String, Object>) taskResponseNode.get("tasks");
+        Map<String, Object> taskResponseStatus = (Map<String, Object>) taskResponseTasks.values().iterator().next();
+        assertThat(ObjectPath.eval("status.job_state", taskResponseStatus), isOneOf(states));
+
+        // check that the rollup job is started using the Cluster State API
+        final Request clusterStateRequest = new Request("GET", "_cluster/state/metadata");
+        Map<String, Object> clusterStateResponse = toMap(client().performRequest(clusterStateRequest));
+        List<Map<String, Object>> rollupJobTasks = ObjectPath.eval("metadata.persistent_tasks.tasks", clusterStateResponse);
+
+        boolean hasRollupTask = false;
+        for (Map<String, Object> task : rollupJobTasks) {
+            if (ObjectPath.eval("id", task).equals(rollupJob)) {
+                hasRollupTask = true;
+
+                final String jobStateField = "task.xpack/rollup/job.state.job_state";
+                assertThat("Expected field [" + jobStateField + "] to be started or indexing in " + task.get("id"),
+                    ObjectPath.eval(jobStateField, task), isOneOf(states));
+                break;
+            }
+        }
+        if (hasRollupTask == false) {
+            fail("Expected persistent task for [" + rollupJob + "] but none found.");
+        }
+
+    }
+
+    private void waitForRollUpJob(final String rollupJob,String[] expectedStates) throws Exception {
+        assertBusy(() -> {
+            final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/" + rollupJob);
+            Response getRollupJobResponse = client().performRequest(getRollupJobRequest);
+            assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+
+            Map<String, Object> job = getJob(getRollupJobResponse, rollupJob);
+            if (job != null) {
+                assertThat(ObjectPath.eval("status.job_state", job), isOneOf(expectedStates));
+            }
+        }, 30L, TimeUnit.SECONDS);
+    }
+
+    private Map<String, Object> getJob(Response response, String targetJobId) throws IOException {
+        return getJob(ESRestTestCase.entityAsMap(response), targetJobId);
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> getJob(Map<String, Object> jobsMap, String targetJobId) throws IOException {
+
+        List<Map<String, Object>> jobs =
+            (List<Map<String, Object>>) XContentMapValues.extractValue("jobs", jobsMap);
+
+        if (jobs == null) {
+            return null;
+        }
+
+        for (Map<String, Object> job : jobs) {
+            String jobId = (String) ((Map<String, Object>) job.get("config")).get("id");
+            if (jobId.equals(targetJobId)) {
+                return job;
+            }
+        }
+        return null;
+    }
+
+    private void waitForPendingTasks() throws Exception {
+        ESTestCase.assertBusy(() -> {
+            try {
+                Request request = new Request("GET", "/_cat/tasks");
+                request.addParameter("detailed", "true");
+                Response response = adminClient().performRequest(request);
+                if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                    try (BufferedReader responseReader = new BufferedReader(
+                        new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
+                        int activeTasks = 0;
+                        String line;
+                        StringBuilder tasksListString = new StringBuilder();
+                        while ((line = responseReader.readLine()) != null) {
+
+                            // We only care about Rollup jobs, otherwise this fails too easily due to unrelated tasks
+                            if (line.startsWith(RollupJob.NAME) == true) {
+                                activeTasks++;
+                                tasksListString.append(line);
+                                tasksListString.append('\n');
+                            }
+                        }
+                        assertEquals(activeTasks + " active tasks found:\n" + tasksListString, 0, activeTasks);
+                    }
+                }
+            } catch (IOException e) {
+                throw new AssertionError("Error getting active tasks list", e);
+            }
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    private void deleteAllJobs() throws Exception {
+        Request request = new Request("GET", "/_xpack/rollup/job/_all");
+        Response response = adminClient().performRequest(request);
+        Map<String, Object> jobs = ESRestTestCase.entityAsMap(response);
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> jobConfigs =
+            (List<Map<String, Object>>) XContentMapValues.extractValue("jobs", jobs);
+
+        if (jobConfigs == null) {
+            return;
+        }
+
+        for (Map<String, Object> jobConfig : jobConfigs) {
+            logger.debug(jobConfig);
+            String jobId = (String) ((Map<String, Object>) jobConfig.get("config")).get("id");
+            logger.debug("Deleting job " + jobId);
+            try {
+                request = new Request("DELETE", "/_xpack/rollup/job/" + jobId);
+                adminClient().performRequest(request);
+            } catch (Exception e) {
+                // ok
+            }
+        }
+    }
+
+    private static String responseEntityToString(Response response) throws Exception {
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
+            return reader.lines().collect(Collectors.joining("\n"));
+        }
+    }
+}

From b1479bbed889787ee9febb3f6b3d3414ba071754 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 16 Jul 2018 18:43:00 +0200
Subject: [PATCH 037/260] Scripting: Remove dead code from painless module
 (#32064)

---
 .../elasticsearch/painless/MethodWriter.java  |  8 ---
 .../painless/antlr/StashingTokenFactory.java  | 62 -------------------
 2 files changed, 70 deletions(-)
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index e0a780d4188..5db7c6b3f71 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -227,14 +227,6 @@ public final class MethodWriter extends GeneratorAdapter {
         return Type.getType(clazz);
     }
 
-    public void writeBranch(final Label tru, final Label fals) {
-        if (tru != null) {
-            visitJumpInsn(Opcodes.IFNE, tru);
-        } else if (fals != null) {
-            visitJumpInsn(Opcodes.IFEQ, fals);
-        }
-    }
-
     /** Starts a new string concat.
      * @return the size of arguments pushed to stack (the object that does string concats, e.g. a StringBuilder)
      */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java
deleted file mode 100644
index 3ac45705d55..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless.antlr;
-
-import org.antlr.v4.runtime.CharStream;
-import org.antlr.v4.runtime.Lexer;
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.TokenFactory;
-import org.antlr.v4.runtime.TokenSource;
-import org.antlr.v4.runtime.misc.Pair;
-
-/**
- * Token factory that preserves that last non-whitespace token so you can do token level lookbehind in the lexer.
- */
-public class StashingTokenFactory<T extends Token> implements TokenFactory<T> {
-    private final TokenFactory<T> delegate;
-
-    private T lastToken;
-
-    public StashingTokenFactory(TokenFactory<T> delegate) {
-        this.delegate = delegate;
-    }
-
-    public T getLastToken() {
-        return lastToken;
-    }
-
-    @Override
-    public T create(Pair<TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line,
-            int charPositionInLine) {
-        return maybeStash(delegate.create(source, type, text, channel, start, stop, line, charPositionInLine));
-    }
-
-    @Override
-    public T create(int type, String text) {
-        return maybeStash(delegate.create(type, text));
-    }
-
-    private T maybeStash(T token) {
-        if (token.getChannel() == Lexer.DEFAULT_TOKEN_CHANNEL) {
-            lastToken = token;
-        }
-        return token;
-    }
-}

From 2a1a28f19c63a268330b3fe351dda731ee0f6627 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 16 Jul 2018 11:15:29 -0700
Subject: [PATCH 038/260] Painless: Separate PainlessLookup into PainlessLookup
 and PainlessLookupBuilder (#32054)

---
 .../painless/AnalyzerCaster.java              |   2 +-
 .../elasticsearch/painless/MethodWriter.java  |   2 +-
 .../painless/PainlessScriptEngine.java        |   8 +-
 .../painless/lookup/PainlessLookup.java       | 692 +---------------
 .../lookup/PainlessLookupBuilder.java         | 774 ++++++++++++++++++
 .../elasticsearch/painless/lookup/def.java    |  28 +
 .../painless/node/EAssignment.java            |   2 +-
 .../elasticsearch/painless/node/EBinary.java  |   2 +-
 .../painless/node/ECapturingFunctionRef.java  |   2 +-
 .../elasticsearch/painless/node/EComp.java    |   2 +-
 .../elasticsearch/painless/node/ELambda.java  |   2 +-
 .../painless/node/EListInit.java              |   2 +-
 .../elasticsearch/painless/node/EMapInit.java |   2 +-
 .../elasticsearch/painless/node/EUnary.java   |   2 +-
 .../elasticsearch/painless/node/PBrace.java   |   2 +-
 .../painless/node/PCallInvoke.java            |   2 +-
 .../elasticsearch/painless/node/PField.java   |   2 +-
 .../painless/node/PSubDefArray.java           |   2 +-
 .../painless/node/PSubDefCall.java            |   2 +-
 .../painless/node/PSubDefField.java           |   2 +-
 .../elasticsearch/painless/node/SEach.java    |   2 +-
 .../painless/node/SSubEachIterable.java       |   2 +-
 .../painless/BaseClassTests.java              |   3 +-
 .../elasticsearch/painless/DebugTests.java    |   3 +-
 .../org/elasticsearch/painless/Debugger.java  |   4 +-
 .../painless/DefBootstrapTests.java           |   3 +-
 .../painless/PainlessDocGenerator.java        |  14 +-
 .../painless/ScriptTestCase.java              |   3 +-
 .../painless/node/NodeToStringTests.java      |   3 +-
 29 files changed, 848 insertions(+), 723 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/def.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 69ef57faad6..457ec82a5e4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessCast;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.Objects;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 5db7c6b3f71..c339e7bfb26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessCast;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
index 4560fd85a65..1687cb24cb6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
@@ -24,7 +24,7 @@ import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.painless.Compiler.Loader;
-import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptContext;
@@ -102,9 +102,11 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
         for (Map.Entry<ScriptContext<?>, List<Whitelist>> entry : contexts.entrySet()) {
             ScriptContext<?> context = entry.getKey();
             if (context.instanceClazz.equals(SearchScript.class) || context.instanceClazz.equals(ExecutableScript.class)) {
-                contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class, new PainlessLookup(entry.getValue())));
+                contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class,
+                    new PainlessLookupBuilder(entry.getValue()).build()));
             } else {
-                contextsToCompilers.put(context, new Compiler(context.instanceClazz, new PainlessLookup(entry.getValue())));
+                contextsToCompilers.put(context, new Compiler(context.instanceClazz,
+                    new PainlessLookupBuilder(entry.getValue()).build()));
             }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
index 5833767fbd3..feeaf4d34bc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
@@ -19,25 +19,10 @@
 
 package org.elasticsearch.painless.lookup;
 
-import org.elasticsearch.painless.spi.Whitelist;
-import org.elasticsearch.painless.spi.WhitelistClass;
-import org.elasticsearch.painless.spi.WhitelistConstructor;
-import org.elasticsearch.painless.spi.WhitelistField;
-import org.elasticsearch.painless.spi.WhitelistMethod;
-import org.objectweb.asm.Type;
-
-import java.lang.invoke.MethodHandle;
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.Stack;
-import java.util.regex.Pattern;
 
 /**
  * The entire API for Painless.  Also used as a whitelist for checking for legal
@@ -45,18 +30,6 @@ import java.util.regex.Pattern;
  */
 public final class PainlessLookup {
 
-    private static final Map<String, PainlessMethod> methodCache = new HashMap<>();
-    private static final Map<String, PainlessField> fieldCache = new HashMap<>();
-
-    private static final Pattern TYPE_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$");
-
-    /** Marker class for def type to be used during type analysis. */
-    public static final class def {
-        private def() {
-
-        }
-    }
-
     public static Class<?> getBoxedType(Class<?> clazz) {
         if (clazz == boolean.class) {
             return Boolean.class;
@@ -205,22 +178,6 @@ public final class PainlessLookup {
         return clazz.getCanonicalName().replace('$', '.');
     }
 
-    private static String buildMethodCacheKey(String structName, String methodName, List<Class<?>> arguments) {
-        StringBuilder key = new StringBuilder();
-        key.append(structName);
-        key.append(methodName);
-
-        for (Class<?> argument : arguments) {
-            key.append(argument.getName());
-        }
-
-        return key.toString();
-    }
-
-    private static String buildFieldCacheKey(String structName, String fieldName, String typeName) {
-        return structName + fieldName + typeName;
-    }
-
     public Collection<PainlessClass> getStructs() {
         return javaClassesToPainlessStructs.values();
     }
@@ -228,652 +185,9 @@ public final class PainlessLookup {
     private final Map<String, Class<?>> painlessTypesToJavaClasses;
     private final Map<Class<?>, PainlessClass> javaClassesToPainlessStructs;
 
-    public PainlessLookup(List<Whitelist> whitelists) {
-        painlessTypesToJavaClasses = new HashMap<>();
-        javaClassesToPainlessStructs = new HashMap<>();
-
-        String origin = null;
-
-        painlessTypesToJavaClasses.put("def", def.class);
-        javaClassesToPainlessStructs.put(def.class, new PainlessClass("def", Object.class, Type.getType(Object.class)));
-
-        try {
-            // first iteration collects all the Painless type names that
-            // are used for validation during the second iteration
-            for (Whitelist whitelist : whitelists) {
-                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
-                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-                    PainlessClass painlessStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(painlessTypeName));
-
-                    if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
-                        throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
-                            "[" + painlessStruct.clazz.getName() + "] and [" + whitelistStruct.javaClassName + "]");
-                    }
-
-                    origin = whitelistStruct.origin;
-                    addStruct(whitelist.javaClassLoader, whitelistStruct);
-
-                    painlessStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(painlessTypeName));
-                    javaClassesToPainlessStructs.put(painlessStruct.clazz, painlessStruct);
-                }
-            }
-
-            // second iteration adds all the constructors, methods, and fields that will
-            // be available in Painless along with validating they exist and all their types have
-            // been white-listed during the first iteration
-            for (Whitelist whitelist : whitelists) {
-                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
-                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-
-                    for (WhitelistConstructor whitelistConstructor : whitelistStruct.whitelistConstructors) {
-                        origin = whitelistConstructor.origin;
-                        addConstructor(painlessTypeName, whitelistConstructor);
-                    }
-
-                    for (WhitelistMethod whitelistMethod : whitelistStruct.whitelistMethods) {
-                        origin = whitelistMethod.origin;
-                        addMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod);
-                    }
-
-                    for (WhitelistField whitelistField : whitelistStruct.whitelistFields) {
-                        origin = whitelistField.origin;
-                        addField(painlessTypeName, whitelistField);
-                    }
-                }
-            }
-        } catch (Exception exception) {
-            throw new IllegalArgumentException("error loading whitelist(s) " + origin, exception);
-        }
-
-        // goes through each Painless struct and determines the inheritance list,
-        // and then adds all inherited types to the Painless struct's whitelist
-        for (Class<?> javaClass : javaClassesToPainlessStructs.keySet()) {
-            PainlessClass painlessStruct = javaClassesToPainlessStructs.get(javaClass);
-
-            List<String> painlessSuperStructs = new ArrayList<>();
-            Class<?> javaSuperClass = painlessStruct.clazz.getSuperclass();
-
-            Stack<Class<?>> javaInteraceLookups = new Stack<>();
-            javaInteraceLookups.push(painlessStruct.clazz);
-
-            // adds super classes to the inheritance list
-            if (javaSuperClass != null && javaSuperClass.isInterface() == false) {
-                while (javaSuperClass != null) {
-                    PainlessClass painlessSuperStruct = javaClassesToPainlessStructs.get(javaSuperClass);
-
-                    if (painlessSuperStruct != null) {
-                        painlessSuperStructs.add(painlessSuperStruct.name);
-                    }
-
-                    javaInteraceLookups.push(javaSuperClass);
-                    javaSuperClass = javaSuperClass.getSuperclass();
-                }
-            }
-
-            // adds all super interfaces to the inheritance list
-            while (javaInteraceLookups.isEmpty() == false) {
-                Class<?> javaInterfaceLookup = javaInteraceLookups.pop();
-
-                for (Class<?> javaSuperInterface : javaInterfaceLookup.getInterfaces()) {
-                    PainlessClass painlessInterfaceStruct = javaClassesToPainlessStructs.get(javaSuperInterface);
-
-                    if (painlessInterfaceStruct != null) {
-                        String painlessInterfaceStructName = painlessInterfaceStruct.name;
-
-                        if (painlessSuperStructs.contains(painlessInterfaceStructName) == false) {
-                            painlessSuperStructs.add(painlessInterfaceStructName);
-                        }
-
-                        for (Class<?> javaPushInterface : javaInterfaceLookup.getInterfaces()) {
-                            javaInteraceLookups.push(javaPushInterface);
-                        }
-                    }
-                }
-            }
-
-            // copies methods and fields from super structs to the parent struct
-            copyStruct(painlessStruct.name, painlessSuperStructs);
-
-            // copies methods and fields from Object into interface types
-            if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) {
-                PainlessClass painlessObjectStruct = javaClassesToPainlessStructs.get(Object.class);
-
-                if (painlessObjectStruct != null) {
-                    copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name));
-                }
-            }
-        }
-
-        // precompute runtime classes
-        for (PainlessClass painlessStruct : javaClassesToPainlessStructs.values()) {
-            addRuntimeClass(painlessStruct);
-        }
-
-        // copy all structs to make them unmodifiable for outside users:
-        for (Map.Entry<Class<?>,PainlessClass> entry : javaClassesToPainlessStructs.entrySet()) {
-            entry.setValue(entry.getValue().freeze(computeFunctionalInterfaceMethod(entry.getValue())));
-        }
-    }
-
-    private void addStruct(ClassLoader whitelistClassLoader, WhitelistClass whitelistStruct) {
-        String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-        String importedPainlessTypeName = painlessTypeName;
-
-        if (TYPE_NAME_PATTERN.matcher(painlessTypeName).matches() == false) {
-            throw new IllegalArgumentException("invalid struct type name [" + painlessTypeName + "]");
-        }
-
-        int index = whitelistStruct.javaClassName.lastIndexOf('.');
-
-        if (index != -1) {
-            importedPainlessTypeName = whitelistStruct.javaClassName.substring(index + 1).replace('$', '.');
-        }
-
-        Class<?> javaClass;
-
-        if      ("void".equals(whitelistStruct.javaClassName))    javaClass = void.class;
-        else if ("boolean".equals(whitelistStruct.javaClassName)) javaClass = boolean.class;
-        else if ("byte".equals(whitelistStruct.javaClassName))    javaClass = byte.class;
-        else if ("short".equals(whitelistStruct.javaClassName))   javaClass = short.class;
-        else if ("char".equals(whitelistStruct.javaClassName))    javaClass = char.class;
-        else if ("int".equals(whitelistStruct.javaClassName))     javaClass = int.class;
-        else if ("long".equals(whitelistStruct.javaClassName))    javaClass = long.class;
-        else if ("float".equals(whitelistStruct.javaClassName))   javaClass = float.class;
-        else if ("double".equals(whitelistStruct.javaClassName))  javaClass = double.class;
-        else {
-            try {
-                javaClass = Class.forName(whitelistStruct.javaClassName, true, whitelistClassLoader);
-            } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("invalid java class name [" + whitelistStruct.javaClassName + "]" +
-                        " for struct [" + painlessTypeName + "]");
-            }
-        }
-
-        PainlessClass existingStruct = javaClassesToPainlessStructs.get(javaClass);
-
-        if (existingStruct == null) {
-            PainlessClass struct = new PainlessClass(painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass));
-            painlessTypesToJavaClasses.put(painlessTypeName, javaClass);
-            javaClassesToPainlessStructs.put(javaClass, struct);
-        } else if (existingStruct.clazz.equals(javaClass) == false) {
-            throw new IllegalArgumentException("struct [" + painlessTypeName + "] is used to " +
-                    "illegally represent multiple java classes [" + whitelistStruct.javaClassName + "] and " +
-                    "[" + existingStruct.clazz.getName() + "]");
-        }
-
-        if (painlessTypeName.equals(importedPainlessTypeName)) {
-            if (whitelistStruct.onlyFQNJavaClassName == false) {
-                throw new IllegalArgumentException("must use only_fqn parameter on type [" + painlessTypeName + "] with no package");
-            }
-        } else {
-            Class<?> importedJavaClass = painlessTypesToJavaClasses.get(importedPainlessTypeName);
-
-            if (importedJavaClass == null) {
-                if (whitelistStruct.onlyFQNJavaClassName == false) {
-                    if (existingStruct != null) {
-                        throw new IllegalArgumentException("inconsistent only_fqn parameters found for type [" + painlessTypeName + "]");
-                    }
-
-                    painlessTypesToJavaClasses.put(importedPainlessTypeName, javaClass);
-                }
-            } else if (importedJavaClass.equals(javaClass) == false) {
-                throw new IllegalArgumentException("imported name [" + painlessTypeName + "] is used to " +
-                    "illegally represent multiple java classes [" + whitelistStruct.javaClassName + "] " +
-                    "and [" + importedJavaClass.getName() + "]");
-            } else if (whitelistStruct.onlyFQNJavaClassName) {
-                throw new IllegalArgumentException("inconsistent only_fqn parameters found for type [" + painlessTypeName + "]");
-            }
-        }
-    }
-
-    private void addConstructor(String ownerStructName, WhitelistConstructor whitelistConstructor) {
-        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
-
-        if (ownerStruct == null) {
-            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for constructor with " +
-                    "parameters " + whitelistConstructor.painlessParameterTypeNames);
-        }
-
-        List<Class<?>> painlessParametersTypes = new ArrayList<>(whitelistConstructor.painlessParameterTypeNames.size());
-        Class<?>[] javaClassParameters = new Class<?>[whitelistConstructor.painlessParameterTypeNames.size()];
-
-        for (int parameterCount = 0; parameterCount < whitelistConstructor.painlessParameterTypeNames.size(); ++parameterCount) {
-            String painlessParameterTypeName = whitelistConstructor.painlessParameterTypeNames.get(parameterCount);
-
-            try {
-                Class<?> painlessParameterClass = getJavaClassFromPainlessType(painlessParameterTypeName);
-
-                painlessParametersTypes.add(painlessParameterClass);
-                javaClassParameters[parameterCount] = defClassToObjectClass(painlessParameterClass);
-            } catch (IllegalArgumentException iae) {
-                throw new IllegalArgumentException("struct not defined for constructor parameter [" + painlessParameterTypeName + "] " +
-                        "with owner struct [" + ownerStructName + "] and constructor parameters " +
-                        whitelistConstructor.painlessParameterTypeNames, iae);
-            }
-        }
-
-        java.lang.reflect.Constructor<?> javaConstructor;
-
-        try {
-            javaConstructor = ownerStruct.clazz.getConstructor(javaClassParameters);
-        } catch (NoSuchMethodException exception) {
-            throw new IllegalArgumentException("constructor not defined for owner struct [" + ownerStructName + "] " +
-                    " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames, exception);
-        }
-
-        PainlessMethodKey painlessMethodKey = new PainlessMethodKey("<init>", whitelistConstructor.painlessParameterTypeNames.size());
-        PainlessMethod painlessConstructor = ownerStruct.constructors.get(painlessMethodKey);
-
-        if (painlessConstructor == null) {
-            org.objectweb.asm.commons.Method asmConstructor = org.objectweb.asm.commons.Method.getMethod(javaConstructor);
-            MethodHandle javaHandle;
-
-            try {
-                javaHandle = MethodHandles.publicLookup().in(ownerStruct.clazz).unreflectConstructor(javaConstructor);
-            } catch (IllegalAccessException exception) {
-                throw new IllegalArgumentException("constructor not defined for owner struct [" + ownerStructName + "] " +
-                        " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames);
-            }
-
-            painlessConstructor = methodCache.computeIfAbsent(buildMethodCacheKey(ownerStruct.name, "<init>", painlessParametersTypes),
-                    key -> new PainlessMethod("<init>", ownerStruct, null, void.class, painlessParametersTypes,
-                            asmConstructor, javaConstructor.getModifiers(), javaHandle));
-            ownerStruct.constructors.put(painlessMethodKey, painlessConstructor);
-        } else if (painlessConstructor.arguments.equals(painlessParametersTypes) == false){
-            throw new IllegalArgumentException(
-                    "illegal duplicate constructors [" + painlessMethodKey + "] found within the struct [" + ownerStruct.name + "] " +
-                    "with parameters " + painlessParametersTypes + " and " + painlessConstructor.arguments);
-        }
-    }
-
-    private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, WhitelistMethod whitelistMethod) {
-        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
-
-        if (ownerStruct == null) {
-            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
-                    "name [" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
-        }
-
-        if (TYPE_NAME_PATTERN.matcher(whitelistMethod.javaMethodName).matches() == false) {
-            throw new IllegalArgumentException("invalid method name" +
-                    " [" + whitelistMethod.javaMethodName + "] for owner struct [" + ownerStructName + "].");
-        }
-
-        Class<?> javaAugmentedClass;
-
-        if (whitelistMethod.javaAugmentedClassName != null) {
-            try {
-                javaAugmentedClass = Class.forName(whitelistMethod.javaAugmentedClassName, true, whitelistClassLoader);
-            } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("augmented class [" + whitelistMethod.javaAugmentedClassName + "] " +
-                        "not found for method with name [" + whitelistMethod.javaMethodName + "] " +
-                        "and parameters " + whitelistMethod.painlessParameterTypeNames, cnfe);
-            }
-        } else {
-            javaAugmentedClass = null;
-        }
-
-        int augmentedOffset = javaAugmentedClass == null ? 0 : 1;
-
-        List<Class<?>> painlessParametersTypes = new ArrayList<>(whitelistMethod.painlessParameterTypeNames.size());
-        Class<?>[] javaClassParameters = new Class<?>[whitelistMethod.painlessParameterTypeNames.size() + augmentedOffset];
-
-        if (javaAugmentedClass != null) {
-            javaClassParameters[0] = ownerStruct.clazz;
-        }
-
-        for (int parameterCount = 0; parameterCount < whitelistMethod.painlessParameterTypeNames.size(); ++parameterCount) {
-            String painlessParameterTypeName = whitelistMethod.painlessParameterTypeNames.get(parameterCount);
-
-            try {
-                Class<?> painlessParameterClass = getJavaClassFromPainlessType(painlessParameterTypeName);
-
-                painlessParametersTypes.add(painlessParameterClass);
-                javaClassParameters[parameterCount + augmentedOffset] = defClassToObjectClass(painlessParameterClass);
-            } catch (IllegalArgumentException iae) {
-                throw new IllegalArgumentException("struct not defined for method parameter [" + painlessParameterTypeName + "] " +
-                        "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
-                        "and parameters " + whitelistMethod.painlessParameterTypeNames, iae);
-            }
-        }
-
-        Class<?> javaImplClass = javaAugmentedClass == null ? ownerStruct.clazz : javaAugmentedClass;
-        java.lang.reflect.Method javaMethod;
-
-        try {
-            javaMethod = javaImplClass.getMethod(whitelistMethod.javaMethodName, javaClassParameters);
-        } catch (NoSuchMethodException nsme) {
-            throw new IllegalArgumentException("method with name [" + whitelistMethod.javaMethodName + "] " +
-                    "and parameters " + whitelistMethod.painlessParameterTypeNames + " not found for class [" +
-                    javaImplClass.getName() + "]", nsme);
-        }
-
-        Class<?> painlessReturnClass;
-
-        try {
-            painlessReturnClass = getJavaClassFromPainlessType(whitelistMethod.painlessReturnTypeName);
-        } catch (IllegalArgumentException iae) {
-            throw new IllegalArgumentException("struct not defined for return type [" + whitelistMethod.painlessReturnTypeName + "] " +
-                    "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
-                    "and parameters " + whitelistMethod.painlessParameterTypeNames, iae);
-        }
-
-        if (javaMethod.getReturnType() != defClassToObjectClass(painlessReturnClass)) {
-            throw new IllegalArgumentException("specified return type class [" + painlessReturnClass + "] " +
-                    "does not match the return type class [" + javaMethod.getReturnType() + "] for the " +
-                    "method with name [" + whitelistMethod.javaMethodName + "] " +
-                    "and parameters " + whitelistMethod.painlessParameterTypeNames);
-        }
-
-        PainlessMethodKey painlessMethodKey =
-                new PainlessMethodKey(whitelistMethod.javaMethodName, whitelistMethod.painlessParameterTypeNames.size());
-
-        if (javaAugmentedClass == null && Modifier.isStatic(javaMethod.getModifiers())) {
-            PainlessMethod painlessMethod = ownerStruct.staticMethods.get(painlessMethodKey);
-
-            if (painlessMethod == null) {
-                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
-                MethodHandle javaMethodHandle;
-
-                try {
-                    javaMethodHandle = MethodHandles.publicLookup().in(javaImplClass).unreflect(javaMethod);
-                } catch (IllegalAccessException exception) {
-                    throw new IllegalArgumentException("method handle not found for method with name " +
-                        "[" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
-                }
-
-                painlessMethod = methodCache.computeIfAbsent(
-                        buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes),
-                        key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct, null, painlessReturnClass,
-                                painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
-                ownerStruct.staticMethods.put(painlessMethodKey, painlessMethod);
-            } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn == painlessReturnClass &&
-                    painlessMethod.arguments.equals(painlessParametersTypes)) == false) {
-                throw new IllegalArgumentException("illegal duplicate static methods [" + painlessMethodKey + "] " +
-                        "found within the struct [" + ownerStruct.name + "] with name [" + whitelistMethod.javaMethodName + "], " +
-                        "return types [" + painlessReturnClass + "] and [" + painlessMethod.rtn + "], " +
-                        "and parameters " + painlessParametersTypes + " and " + painlessMethod.arguments);
-            }
-        } else {
-            PainlessMethod painlessMethod = ownerStruct.methods.get(painlessMethodKey);
-
-            if (painlessMethod == null) {
-                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
-                MethodHandle javaMethodHandle;
-
-                try {
-                    javaMethodHandle = MethodHandles.publicLookup().in(javaImplClass).unreflect(javaMethod);
-                } catch (IllegalAccessException exception) {
-                    throw new IllegalArgumentException("method handle not found for method with name " +
-                        "[" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
-                }
-
-                painlessMethod = methodCache.computeIfAbsent(
-                        buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes),
-                        key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct, javaAugmentedClass, painlessReturnClass,
-                                painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
-                ownerStruct.methods.put(painlessMethodKey, painlessMethod);
-            } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn.equals(painlessReturnClass) &&
-                painlessMethod.arguments.equals(painlessParametersTypes)) == false) {
-                throw new IllegalArgumentException("illegal duplicate member methods [" + painlessMethodKey + "] " +
-                    "found within the struct [" + ownerStruct.name + "] with name [" + whitelistMethod.javaMethodName + "], " +
-                    "return types [" + painlessReturnClass + "] and [" + painlessMethod.rtn + "], " +
-                    "and parameters " + painlessParametersTypes + " and " + painlessMethod.arguments);
-            }
-        }
-    }
-
-    private void addField(String ownerStructName, WhitelistField whitelistField) {
-        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
-
-        if (ownerStruct == null) {
-            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
-                    "name [" + whitelistField.javaFieldName + "] and type " + whitelistField.painlessFieldTypeName);
-        }
-
-        if (TYPE_NAME_PATTERN.matcher(whitelistField.javaFieldName).matches() == false) {
-            throw new IllegalArgumentException("invalid field name " +
-                    "[" + whitelistField.painlessFieldTypeName + "] for owner struct [" + ownerStructName + "].");
-        }
-
-        java.lang.reflect.Field javaField;
-
-        try {
-            javaField = ownerStruct.clazz.getField(whitelistField.javaFieldName);
-        } catch (NoSuchFieldException exception) {
-            throw new IllegalArgumentException("field [" + whitelistField.javaFieldName + "] " +
-                    "not found for class [" + ownerStruct.clazz.getName() + "].");
-        }
-
-        Class<?> painlessFieldClass;
-
-        try {
-            painlessFieldClass = getJavaClassFromPainlessType(whitelistField.painlessFieldTypeName);
-        } catch (IllegalArgumentException iae) {
-            throw new IllegalArgumentException("struct not defined for return type [" + whitelistField.painlessFieldTypeName + "] " +
-                "with owner struct [" + ownerStructName + "] and field with name [" + whitelistField.javaFieldName + "]", iae);
-        }
-
-        if (Modifier.isStatic(javaField.getModifiers())) {
-            if (Modifier.isFinal(javaField.getModifiers()) == false) {
-                throw new IllegalArgumentException("static [" + whitelistField.javaFieldName + "] " +
-                        "with owner struct [" + ownerStruct.name + "] is not final");
-            }
-
-            PainlessField painlessField = ownerStruct.staticMembers.get(whitelistField.javaFieldName);
-
-            if (painlessField == null) {
-                painlessField = fieldCache.computeIfAbsent(
-                        buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()),
-                        key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
-                                ownerStruct, painlessFieldClass, javaField.getModifiers(), null, null));
-                ownerStruct.staticMembers.put(whitelistField.javaFieldName, painlessField);
-            } else if (painlessField.clazz != painlessFieldClass) {
-                throw new IllegalArgumentException("illegal duplicate static fields [" + whitelistField.javaFieldName + "] " +
-                    "found within the struct [" + ownerStruct.name + "] with type [" + whitelistField.painlessFieldTypeName + "]");
-            }
-        } else {
-            MethodHandle javaMethodHandleGetter;
-            MethodHandle javaMethodHandleSetter;
-
-            try {
-                if (Modifier.isStatic(javaField.getModifiers()) == false) {
-                    javaMethodHandleGetter = MethodHandles.publicLookup().unreflectGetter(javaField);
-                    javaMethodHandleSetter = MethodHandles.publicLookup().unreflectSetter(javaField);
-                } else {
-                    javaMethodHandleGetter = null;
-                    javaMethodHandleSetter = null;
-                }
-            } catch (IllegalAccessException exception) {
-                throw new IllegalArgumentException("getter/setter [" + whitelistField.javaFieldName + "]" +
-                    " not found for class [" + ownerStruct.clazz.getName() + "].");
-            }
-
-            PainlessField painlessField = ownerStruct.members.get(whitelistField.javaFieldName);
-
-            if (painlessField == null) {
-                painlessField = fieldCache.computeIfAbsent(
-                        buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()),
-                        key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
-                                ownerStruct, painlessFieldClass, javaField.getModifiers(), javaMethodHandleGetter, javaMethodHandleSetter));
-                ownerStruct.members.put(whitelistField.javaFieldName, painlessField);
-            } else if (painlessField.clazz != painlessFieldClass) {
-                throw new IllegalArgumentException("illegal duplicate member fields [" + whitelistField.javaFieldName + "] " +
-                    "found within the struct [" + ownerStruct.name + "] with type [" + whitelistField.painlessFieldTypeName + "]");
-            }
-        }
-    }
-
-    private void copyStruct(String struct, List<String> children) {
-        final PainlessClass owner = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(struct));
-
-        if (owner == null) {
-            throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
-        }
-
-        for (int count = 0; count < children.size(); ++count) {
-            final PainlessClass child = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(children.get(count)));
-
-            if (child == null) {
-                throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
-                    " not defined for copy to owner struct [" + owner.name + "].");
-            }
-
-            if (!child.clazz.isAssignableFrom(owner.clazz)) {
-                throw new ClassCastException("Child struct [" + child.name + "]" +
-                    " is not a super type of owner struct [" + owner.name + "] in copy.");
-            }
-
-            for (Map.Entry<PainlessMethodKey,PainlessMethod> kvPair : child.methods.entrySet()) {
-                PainlessMethodKey methodKey = kvPair.getKey();
-                PainlessMethod method = kvPair.getValue();
-                if (owner.methods.get(methodKey) == null) {
-                    // TODO: some of these are no longer valid or outright don't work
-                    // TODO: since classes may not come from the Painless classloader
-                    // TODO: and it was dependent on the order of the extends which
-                    // TODO: which no longer exists since this is generated automatically
-                    // sanity check, look for missing covariant/generic override
-                    /*if (owner.clazz.isInterface() && child.clazz == Object.class) {
-                        // ok
-                    } else if (child.clazz == Spliterator.OfPrimitive.class || child.clazz == PrimitiveIterator.class) {
-                        // ok, we rely on generics erasure for these (its guaranteed in the javadocs though!!!!)
-                    } else if (Constants.JRE_IS_MINIMUM_JAVA9 && owner.clazz == LocalDate.class) {
-                        // ok, java 9 added covariant override for LocalDate.getEra() to return IsoEra:
-                        // https://bugs.openjdk.java.net/browse/JDK-8072746
-                    } else {
-                        try {
-                            // TODO: we *have* to remove all these public members and use getter methods to encapsulate!
-                            final Class<?> impl;
-                            final Class<?> arguments[];
-                            if (method.augmentation != null) {
-                                impl = method.augmentation;
-                                arguments = new Class<?>[method.arguments.size() + 1];
-                                arguments[0] = method.owner.clazz;
-                                for (int i = 0; i < method.arguments.size(); i++) {
-                                    arguments[i + 1] = method.arguments.get(i).clazz;
-                                }
-                            } else {
-                                impl = owner.clazz;
-                                arguments = new Class<?>[method.arguments.size()];
-                                for (int i = 0; i < method.arguments.size(); i++) {
-                                    arguments[i] = method.arguments.get(i).clazz;
-                                }
-                            }
-                            java.lang.reflect.Method m = impl.getMethod(method.method.getName(), arguments);
-                            if (m.getReturnType() != method.rtn.clazz) {
-                                throw new IllegalStateException("missing covariant override for: " + m + " in " + owner.name);
-                            }
-                            if (m.isBridge() && !Modifier.isVolatile(method.modifiers)) {
-                                // its a bridge in the destination, but not in the source, but it might still be ok, check generics:
-                                java.lang.reflect.Method source = child.clazz.getMethod(method.method.getName(), arguments);
-                                if (!Arrays.equals(source.getGenericParameterTypes(), source.getParameterTypes())) {
-                                    throw new IllegalStateException("missing generic override for: " + m + " in " + owner.name);
-                                }
-                            }
-                        } catch (ReflectiveOperationException e) {
-                            throw new AssertionError(e);
-                        }
-                    }*/
-                    owner.methods.put(methodKey, method);
-                }
-            }
-
-            for (PainlessField field : child.members.values()) {
-                if (owner.members.get(field.name) == null) {
-                    owner.members.put(field.name,
-                        new PainlessField(field.name, field.javaName, owner, field.clazz, field.modifiers, field.getter, field.setter));
-                }
-            }
-        }
-    }
-
-    /**
-     * Precomputes a more efficient structure for dynamic method/field access.
-     */
-    private void addRuntimeClass(final PainlessClass struct) {
-        // add all getters/setters
-        for (Map.Entry<PainlessMethodKey, PainlessMethod> method : struct.methods.entrySet()) {
-            String name = method.getKey().name;
-            PainlessMethod m = method.getValue();
-
-            if (m.arguments.size() == 0 &&
-                name.startsWith("get") &&
-                name.length() > 3 &&
-                Character.isUpperCase(name.charAt(3))) {
-                StringBuilder newName = new StringBuilder();
-                newName.append(Character.toLowerCase(name.charAt(3)));
-                newName.append(name.substring(4));
-                struct.getters.putIfAbsent(newName.toString(), m.handle);
-            } else if (m.arguments.size() == 0 &&
-                name.startsWith("is") &&
-                name.length() > 2 &&
-                Character.isUpperCase(name.charAt(2))) {
-                StringBuilder newName = new StringBuilder();
-                newName.append(Character.toLowerCase(name.charAt(2)));
-                newName.append(name.substring(3));
-                struct.getters.putIfAbsent(newName.toString(), m.handle);
-            }
-
-            if (m.arguments.size() == 1 &&
-                name.startsWith("set") &&
-                name.length() > 3 &&
-                Character.isUpperCase(name.charAt(3))) {
-                StringBuilder newName = new StringBuilder();
-                newName.append(Character.toLowerCase(name.charAt(3)));
-                newName.append(name.substring(4));
-                struct.setters.putIfAbsent(newName.toString(), m.handle);
-            }
-        }
-
-        // add all members
-        for (Map.Entry<String, PainlessField> member : struct.members.entrySet()) {
-            struct.getters.put(member.getKey(), member.getValue().getter);
-            struct.setters.put(member.getKey(), member.getValue().setter);
-        }
-    }
-
-    /** computes the functional interface method for a class, or returns null */
-    private PainlessMethod computeFunctionalInterfaceMethod(PainlessClass clazz) {
-        if (!clazz.clazz.isInterface()) {
-            return null;
-        }
-        // if its marked with this annotation, we fail if the conditions don't hold (means whitelist bug)
-        // otherwise, this annotation is pretty useless.
-        boolean hasAnnotation = clazz.clazz.isAnnotationPresent(FunctionalInterface.class);
-        List<java.lang.reflect.Method> methods = new ArrayList<>();
-        for (java.lang.reflect.Method m : clazz.clazz.getMethods()) {
-            // default interface methods don't count
-            if (m.isDefault()) {
-                continue;
-            }
-            // static methods don't count
-            if (Modifier.isStatic(m.getModifiers())) {
-                continue;
-            }
-            // if its from Object, it doesn't count
-            try {
-                Object.class.getMethod(m.getName(), m.getParameterTypes());
-                continue;
-            } catch (ReflectiveOperationException e) {
-                // it counts
-            }
-            methods.add(m);
-        }
-        if (methods.size() != 1) {
-            if (hasAnnotation) {
-                throw new IllegalArgumentException("Class: " + clazz.name +
-                    " is marked with FunctionalInterface but doesn't fit the bill: " + methods);
-            }
-            return null;
-        }
-        // inspect the one method found from the reflection API, it should match the whitelist!
-        java.lang.reflect.Method oneMethod = methods.get(0);
-        PainlessMethod painless = clazz.methods.get(new PainlessMethodKey(oneMethod.getName(), oneMethod.getParameterCount()));
-        if (painless == null || painless.method.equals(org.objectweb.asm.commons.Method.getMethod(oneMethod)) == false) {
-            throw new IllegalArgumentException("Class: " + clazz.name + " is functional but the functional " +
-                "method is not whitelisted!");
-        }
-        return painless;
+    PainlessLookup(Map<String, Class<?>> painlessTypesToJavaClasses, Map<Class<?>, PainlessClass> javaClassesToPainlessStructs) {
+        this.painlessTypesToJavaClasses = Collections.unmodifiableMap(painlessTypesToJavaClasses);
+        this.javaClassesToPainlessStructs = Collections.unmodifiableMap(javaClassesToPainlessStructs);
     }
 
     public boolean isSimplePainlessType(String painlessType) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
new file mode 100644
index 00000000000..1dadce318d6
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -0,0 +1,774 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.lookup;
+
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistClass;
+import org.elasticsearch.painless.spi.WhitelistConstructor;
+import org.elasticsearch.painless.spi.WhitelistField;
+import org.elasticsearch.painless.spi.WhitelistMethod;
+import org.objectweb.asm.Type;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+import java.util.regex.Pattern;
+
+public class PainlessLookupBuilder {
+    private static final Pattern TYPE_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$");
+
+    private static final Map<String, PainlessMethod> methodCache = new HashMap<>();
+    private static final Map<String, PainlessField> fieldCache = new HashMap<>();
+
+    private static String buildMethodCacheKey(String structName, String methodName, List<Class<?>> arguments) {
+        StringBuilder key = new StringBuilder();
+        key.append(structName);
+        key.append(methodName);
+
+        for (Class<?> argument : arguments) {
+            key.append(argument.getName());
+        }
+
+        return key.toString();
+    }
+
+    private static String buildFieldCacheKey(String structName, String fieldName, String typeName) {
+        return structName + fieldName + typeName;
+    }
+
+    private final Map<String, Class<?>> painlessTypesToJavaClasses;
+    private final Map<Class<?>, PainlessClass> javaClassesToPainlessStructs;
+
+    public PainlessLookupBuilder(List<Whitelist> whitelists) {
+        painlessTypesToJavaClasses = new HashMap<>();
+        javaClassesToPainlessStructs = new HashMap<>();
+
+        String origin = null;
+
+        painlessTypesToJavaClasses.put("def", def.class);
+        javaClassesToPainlessStructs.put(def.class, new PainlessClass("def", Object.class, Type.getType(Object.class)));
+
+        try {
+            // first iteration collects all the Painless type names that
+            // are used for validation during the second iteration
+            for (Whitelist whitelist : whitelists) {
+                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
+                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
+                    PainlessClass painlessStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(painlessTypeName));
+
+                    if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
+                        throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
+                            "[" + painlessStruct.clazz.getName() + "] and [" + whitelistStruct.javaClassName + "]");
+                    }
+
+                    origin = whitelistStruct.origin;
+                    addStruct(whitelist.javaClassLoader, whitelistStruct);
+
+                    painlessStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(painlessTypeName));
+                    javaClassesToPainlessStructs.put(painlessStruct.clazz, painlessStruct);
+                }
+            }
+
+            // second iteration adds all the constructors, methods, and fields that will
+            // be available in Painless along with validating they exist and all their types have
+            // been white-listed during the first iteration
+            for (Whitelist whitelist : whitelists) {
+                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
+                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
+
+                    for (WhitelistConstructor whitelistConstructor : whitelistStruct.whitelistConstructors) {
+                        origin = whitelistConstructor.origin;
+                        addConstructor(painlessTypeName, whitelistConstructor);
+                    }
+
+                    for (WhitelistMethod whitelistMethod : whitelistStruct.whitelistMethods) {
+                        origin = whitelistMethod.origin;
+                        addMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod);
+                    }
+
+                    for (WhitelistField whitelistField : whitelistStruct.whitelistFields) {
+                        origin = whitelistField.origin;
+                        addField(painlessTypeName, whitelistField);
+                    }
+                }
+            }
+        } catch (Exception exception) {
+            throw new IllegalArgumentException("error loading whitelist(s) " + origin, exception);
+        }
+
+        // goes through each Painless struct and determines the inheritance list,
+        // and then adds all inherited types to the Painless struct's whitelist
+        for (Class<?> javaClass : javaClassesToPainlessStructs.keySet()) {
+            PainlessClass painlessStruct = javaClassesToPainlessStructs.get(javaClass);
+
+            List<String> painlessSuperStructs = new ArrayList<>();
+            Class<?> javaSuperClass = painlessStruct.clazz.getSuperclass();
+
+            Stack<Class<?>> javaInteraceLookups = new Stack<>();
+            javaInteraceLookups.push(painlessStruct.clazz);
+
+            // adds super classes to the inheritance list
+            if (javaSuperClass != null && javaSuperClass.isInterface() == false) {
+                while (javaSuperClass != null) {
+                    PainlessClass painlessSuperStruct = javaClassesToPainlessStructs.get(javaSuperClass);
+
+                    if (painlessSuperStruct != null) {
+                        painlessSuperStructs.add(painlessSuperStruct.name);
+                    }
+
+                    javaInteraceLookups.push(javaSuperClass);
+                    javaSuperClass = javaSuperClass.getSuperclass();
+                }
+            }
+
+            // adds all super interfaces to the inheritance list
+            while (javaInteraceLookups.isEmpty() == false) {
+                Class<?> javaInterfaceLookup = javaInteraceLookups.pop();
+
+                for (Class<?> javaSuperInterface : javaInterfaceLookup.getInterfaces()) {
+                    PainlessClass painlessInterfaceStruct = javaClassesToPainlessStructs.get(javaSuperInterface);
+
+                    if (painlessInterfaceStruct != null) {
+                        String painlessInterfaceStructName = painlessInterfaceStruct.name;
+
+                        if (painlessSuperStructs.contains(painlessInterfaceStructName) == false) {
+                            painlessSuperStructs.add(painlessInterfaceStructName);
+                        }
+
+                        for (Class<?> javaPushInterface : javaInterfaceLookup.getInterfaces()) {
+                            javaInteraceLookups.push(javaPushInterface);
+                        }
+                    }
+                }
+            }
+
+            // copies methods and fields from super structs to the parent struct
+            copyStruct(painlessStruct.name, painlessSuperStructs);
+
+            // copies methods and fields from Object into interface types
+            if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) {
+                PainlessClass painlessObjectStruct = javaClassesToPainlessStructs.get(Object.class);
+
+                if (painlessObjectStruct != null) {
+                    copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name));
+                }
+            }
+        }
+
+        // precompute runtime classes
+        for (PainlessClass painlessStruct : javaClassesToPainlessStructs.values()) {
+            addRuntimeClass(painlessStruct);
+        }
+
+        // copy all structs to make them unmodifiable for outside users:
+        for (Map.Entry<Class<?>,PainlessClass> entry : javaClassesToPainlessStructs.entrySet()) {
+            entry.setValue(entry.getValue().freeze(computeFunctionalInterfaceMethod(entry.getValue())));
+        }
+    }
+
+    private void addStruct(ClassLoader whitelistClassLoader, WhitelistClass whitelistStruct) {
+        String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
+        String importedPainlessTypeName = painlessTypeName;
+
+        if (TYPE_NAME_PATTERN.matcher(painlessTypeName).matches() == false) {
+            throw new IllegalArgumentException("invalid struct type name [" + painlessTypeName + "]");
+        }
+
+        int index = whitelistStruct.javaClassName.lastIndexOf('.');
+
+        if (index != -1) {
+            importedPainlessTypeName = whitelistStruct.javaClassName.substring(index + 1).replace('$', '.');
+        }
+
+        Class<?> javaClass;
+
+        if      ("void".equals(whitelistStruct.javaClassName))    javaClass = void.class;
+        else if ("boolean".equals(whitelistStruct.javaClassName)) javaClass = boolean.class;
+        else if ("byte".equals(whitelistStruct.javaClassName))    javaClass = byte.class;
+        else if ("short".equals(whitelistStruct.javaClassName))   javaClass = short.class;
+        else if ("char".equals(whitelistStruct.javaClassName))    javaClass = char.class;
+        else if ("int".equals(whitelistStruct.javaClassName))     javaClass = int.class;
+        else if ("long".equals(whitelistStruct.javaClassName))    javaClass = long.class;
+        else if ("float".equals(whitelistStruct.javaClassName))   javaClass = float.class;
+        else if ("double".equals(whitelistStruct.javaClassName))  javaClass = double.class;
+        else {
+            try {
+                javaClass = Class.forName(whitelistStruct.javaClassName, true, whitelistClassLoader);
+            } catch (ClassNotFoundException cnfe) {
+                throw new IllegalArgumentException("invalid java class name [" + whitelistStruct.javaClassName + "]" +
+                    " for struct [" + painlessTypeName + "]");
+            }
+        }
+
+        PainlessClass existingStruct = javaClassesToPainlessStructs.get(javaClass);
+
+        if (existingStruct == null) {
+            PainlessClass struct = new PainlessClass(painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass));
+            painlessTypesToJavaClasses.put(painlessTypeName, javaClass);
+            javaClassesToPainlessStructs.put(javaClass, struct);
+        } else if (existingStruct.clazz.equals(javaClass) == false) {
+            throw new IllegalArgumentException("struct [" + painlessTypeName + "] is used to " +
+                "illegally represent multiple java classes [" + whitelistStruct.javaClassName + "] and " +
+                "[" + existingStruct.clazz.getName() + "]");
+        }
+
+        if (painlessTypeName.equals(importedPainlessTypeName)) {
+            if (whitelistStruct.onlyFQNJavaClassName == false) {
+                throw new IllegalArgumentException("must use only_fqn parameter on type [" + painlessTypeName + "] with no package");
+            }
+        } else {
+            Class<?> importedJavaClass = painlessTypesToJavaClasses.get(importedPainlessTypeName);
+
+            if (importedJavaClass == null) {
+                if (whitelistStruct.onlyFQNJavaClassName == false) {
+                    if (existingStruct != null) {
+                        throw new IllegalArgumentException("inconsistent only_fqn parameters found for type [" + painlessTypeName + "]");
+                    }
+
+                    painlessTypesToJavaClasses.put(importedPainlessTypeName, javaClass);
+                }
+            } else if (importedJavaClass.equals(javaClass) == false) {
+                throw new IllegalArgumentException("imported name [" + painlessTypeName + "] is used to " +
+                    "illegally represent multiple java classes [" + whitelistStruct.javaClassName + "] " +
+                    "and [" + importedJavaClass.getName() + "]");
+            } else if (whitelistStruct.onlyFQNJavaClassName) {
+                throw new IllegalArgumentException("inconsistent only_fqn parameters found for type [" + painlessTypeName + "]");
+            }
+        }
+    }
+
+    private void addConstructor(String ownerStructName, WhitelistConstructor whitelistConstructor) {
+        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
+
+        if (ownerStruct == null) {
+            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for constructor with " +
+                "parameters " + whitelistConstructor.painlessParameterTypeNames);
+        }
+
+        List<Class<?>> painlessParametersTypes = new ArrayList<>(whitelistConstructor.painlessParameterTypeNames.size());
+        Class<?>[] javaClassParameters = new Class<?>[whitelistConstructor.painlessParameterTypeNames.size()];
+
+        for (int parameterCount = 0; parameterCount < whitelistConstructor.painlessParameterTypeNames.size(); ++parameterCount) {
+            String painlessParameterTypeName = whitelistConstructor.painlessParameterTypeNames.get(parameterCount);
+
+            try {
+                Class<?> painlessParameterClass = getJavaClassFromPainlessType(painlessParameterTypeName);
+
+                painlessParametersTypes.add(painlessParameterClass);
+                javaClassParameters[parameterCount] = PainlessLookup.defClassToObjectClass(painlessParameterClass);
+            } catch (IllegalArgumentException iae) {
+                throw new IllegalArgumentException("struct not defined for constructor parameter [" + painlessParameterTypeName + "] " +
+                    "with owner struct [" + ownerStructName + "] and constructor parameters " +
+                    whitelistConstructor.painlessParameterTypeNames, iae);
+            }
+        }
+
+        java.lang.reflect.Constructor<?> javaConstructor;
+
+        try {
+            javaConstructor = ownerStruct.clazz.getConstructor(javaClassParameters);
+        } catch (NoSuchMethodException exception) {
+            throw new IllegalArgumentException("constructor not defined for owner struct [" + ownerStructName + "] " +
+                " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames, exception);
+        }
+
+        PainlessMethodKey painlessMethodKey = new PainlessMethodKey("<init>", whitelistConstructor.painlessParameterTypeNames.size());
+        PainlessMethod painlessConstructor = ownerStruct.constructors.get(painlessMethodKey);
+
+        if (painlessConstructor == null) {
+            org.objectweb.asm.commons.Method asmConstructor = org.objectweb.asm.commons.Method.getMethod(javaConstructor);
+            MethodHandle javaHandle;
+
+            try {
+                javaHandle = MethodHandles.publicLookup().in(ownerStruct.clazz).unreflectConstructor(javaConstructor);
+            } catch (IllegalAccessException exception) {
+                throw new IllegalArgumentException("constructor not defined for owner struct [" + ownerStructName + "] " +
+                    " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames);
+            }
+
+            painlessConstructor = methodCache.computeIfAbsent(buildMethodCacheKey(ownerStruct.name, "<init>", painlessParametersTypes),
+                key -> new PainlessMethod("<init>", ownerStruct, null, void.class, painlessParametersTypes,
+                    asmConstructor, javaConstructor.getModifiers(), javaHandle));
+            ownerStruct.constructors.put(painlessMethodKey, painlessConstructor);
+        } else if (painlessConstructor.arguments.equals(painlessParametersTypes) == false){
+            throw new IllegalArgumentException(
+                "illegal duplicate constructors [" + painlessMethodKey + "] found within the struct [" + ownerStruct.name + "] " +
+                    "with parameters " + painlessParametersTypes + " and " + painlessConstructor.arguments);
+        }
+    }
+
+    private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, WhitelistMethod whitelistMethod) {
+        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
+
+        if (ownerStruct == null) {
+            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
+                "name [" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
+        }
+
+        if (TYPE_NAME_PATTERN.matcher(whitelistMethod.javaMethodName).matches() == false) {
+            throw new IllegalArgumentException("invalid method name" +
+                " [" + whitelistMethod.javaMethodName + "] for owner struct [" + ownerStructName + "].");
+        }
+
+        Class<?> javaAugmentedClass;
+
+        if (whitelistMethod.javaAugmentedClassName != null) {
+            try {
+                javaAugmentedClass = Class.forName(whitelistMethod.javaAugmentedClassName, true, whitelistClassLoader);
+            } catch (ClassNotFoundException cnfe) {
+                throw new IllegalArgumentException("augmented class [" + whitelistMethod.javaAugmentedClassName + "] " +
+                    "not found for method with name [" + whitelistMethod.javaMethodName + "] " +
+                    "and parameters " + whitelistMethod.painlessParameterTypeNames, cnfe);
+            }
+        } else {
+            javaAugmentedClass = null;
+        }
+
+        int augmentedOffset = javaAugmentedClass == null ? 0 : 1;
+
+        List<Class<?>> painlessParametersTypes = new ArrayList<>(whitelistMethod.painlessParameterTypeNames.size());
+        Class<?>[] javaClassParameters = new Class<?>[whitelistMethod.painlessParameterTypeNames.size() + augmentedOffset];
+
+        if (javaAugmentedClass != null) {
+            javaClassParameters[0] = ownerStruct.clazz;
+        }
+
+        for (int parameterCount = 0; parameterCount < whitelistMethod.painlessParameterTypeNames.size(); ++parameterCount) {
+            String painlessParameterTypeName = whitelistMethod.painlessParameterTypeNames.get(parameterCount);
+
+            try {
+                Class<?> painlessParameterClass = getJavaClassFromPainlessType(painlessParameterTypeName);
+
+                painlessParametersTypes.add(painlessParameterClass);
+                javaClassParameters[parameterCount + augmentedOffset] = PainlessLookup.defClassToObjectClass(painlessParameterClass);
+            } catch (IllegalArgumentException iae) {
+                throw new IllegalArgumentException("struct not defined for method parameter [" + painlessParameterTypeName + "] " +
+                    "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
+                    "and parameters " + whitelistMethod.painlessParameterTypeNames, iae);
+            }
+        }
+
+        Class<?> javaImplClass = javaAugmentedClass == null ? ownerStruct.clazz : javaAugmentedClass;
+        java.lang.reflect.Method javaMethod;
+
+        try {
+            javaMethod = javaImplClass.getMethod(whitelistMethod.javaMethodName, javaClassParameters);
+        } catch (NoSuchMethodException nsme) {
+            throw new IllegalArgumentException("method with name [" + whitelistMethod.javaMethodName + "] " +
+                "and parameters " + whitelistMethod.painlessParameterTypeNames + " not found for class [" +
+                javaImplClass.getName() + "]", nsme);
+        }
+
+        Class<?> painlessReturnClass;
+
+        try {
+            painlessReturnClass = getJavaClassFromPainlessType(whitelistMethod.painlessReturnTypeName);
+        } catch (IllegalArgumentException iae) {
+            throw new IllegalArgumentException("struct not defined for return type [" + whitelistMethod.painlessReturnTypeName + "] " +
+                "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
+                "and parameters " + whitelistMethod.painlessParameterTypeNames, iae);
+        }
+
+        if (javaMethod.getReturnType() != PainlessLookup.defClassToObjectClass(painlessReturnClass)) {
+            throw new IllegalArgumentException("specified return type class [" + painlessReturnClass + "] " +
+                "does not match the return type class [" + javaMethod.getReturnType() + "] for the " +
+                "method with name [" + whitelistMethod.javaMethodName + "] " +
+                "and parameters " + whitelistMethod.painlessParameterTypeNames);
+        }
+
+        PainlessMethodKey painlessMethodKey =
+            new PainlessMethodKey(whitelistMethod.javaMethodName, whitelistMethod.painlessParameterTypeNames.size());
+
+        if (javaAugmentedClass == null && Modifier.isStatic(javaMethod.getModifiers())) {
+            PainlessMethod painlessMethod = ownerStruct.staticMethods.get(painlessMethodKey);
+
+            if (painlessMethod == null) {
+                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
+                MethodHandle javaMethodHandle;
+
+                try {
+                    javaMethodHandle = MethodHandles.publicLookup().in(javaImplClass).unreflect(javaMethod);
+                } catch (IllegalAccessException exception) {
+                    throw new IllegalArgumentException("method handle not found for method with name " +
+                        "[" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
+                }
+
+                painlessMethod = methodCache.computeIfAbsent(
+                    buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes),
+                    key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct, null, painlessReturnClass,
+                        painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
+                ownerStruct.staticMethods.put(painlessMethodKey, painlessMethod);
+            } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn == painlessReturnClass &&
+                painlessMethod.arguments.equals(painlessParametersTypes)) == false) {
+                throw new IllegalArgumentException("illegal duplicate static methods [" + painlessMethodKey + "] " +
+                    "found within the struct [" + ownerStruct.name + "] with name [" + whitelistMethod.javaMethodName + "], " +
+                    "return types [" + painlessReturnClass + "] and [" + painlessMethod.rtn + "], " +
+                    "and parameters " + painlessParametersTypes + " and " + painlessMethod.arguments);
+            }
+        } else {
+            PainlessMethod painlessMethod = ownerStruct.methods.get(painlessMethodKey);
+
+            if (painlessMethod == null) {
+                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
+                MethodHandle javaMethodHandle;
+
+                try {
+                    javaMethodHandle = MethodHandles.publicLookup().in(javaImplClass).unreflect(javaMethod);
+                } catch (IllegalAccessException exception) {
+                    throw new IllegalArgumentException("method handle not found for method with name " +
+                        "[" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
+                }
+
+                painlessMethod = methodCache.computeIfAbsent(
+                    buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes),
+                    key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct, javaAugmentedClass, painlessReturnClass,
+                        painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
+                ownerStruct.methods.put(painlessMethodKey, painlessMethod);
+            } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn.equals(painlessReturnClass) &&
+                painlessMethod.arguments.equals(painlessParametersTypes)) == false) {
+                throw new IllegalArgumentException("illegal duplicate member methods [" + painlessMethodKey + "] " +
+                    "found within the struct [" + ownerStruct.name + "] with name [" + whitelistMethod.javaMethodName + "], " +
+                    "return types [" + painlessReturnClass + "] and [" + painlessMethod.rtn + "], " +
+                    "and parameters " + painlessParametersTypes + " and " + painlessMethod.arguments);
+            }
+        }
+    }
+
+    private void addField(String ownerStructName, WhitelistField whitelistField) {
+        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
+
+        if (ownerStruct == null) {
+            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
+                "name [" + whitelistField.javaFieldName + "] and type " + whitelistField.painlessFieldTypeName);
+        }
+
+        if (TYPE_NAME_PATTERN.matcher(whitelistField.javaFieldName).matches() == false) {
+            throw new IllegalArgumentException("invalid field name " +
+                "[" + whitelistField.painlessFieldTypeName + "] for owner struct [" + ownerStructName + "].");
+        }
+
+        java.lang.reflect.Field javaField;
+
+        try {
+            javaField = ownerStruct.clazz.getField(whitelistField.javaFieldName);
+        } catch (NoSuchFieldException exception) {
+            throw new IllegalArgumentException("field [" + whitelistField.javaFieldName + "] " +
+                "not found for class [" + ownerStruct.clazz.getName() + "].");
+        }
+
+        Class<?> painlessFieldClass;
+
+        try {
+            painlessFieldClass = getJavaClassFromPainlessType(whitelistField.painlessFieldTypeName);
+        } catch (IllegalArgumentException iae) {
+            throw new IllegalArgumentException("struct not defined for return type [" + whitelistField.painlessFieldTypeName + "] " +
+                "with owner struct [" + ownerStructName + "] and field with name [" + whitelistField.javaFieldName + "]", iae);
+        }
+
+        if (Modifier.isStatic(javaField.getModifiers())) {
+            if (Modifier.isFinal(javaField.getModifiers()) == false) {
+                throw new IllegalArgumentException("static [" + whitelistField.javaFieldName + "] " +
+                    "with owner struct [" + ownerStruct.name + "] is not final");
+            }
+
+            PainlessField painlessField = ownerStruct.staticMembers.get(whitelistField.javaFieldName);
+
+            if (painlessField == null) {
+                painlessField = fieldCache.computeIfAbsent(
+                    buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()),
+                    key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
+                        ownerStruct, painlessFieldClass, javaField.getModifiers(), null, null));
+                ownerStruct.staticMembers.put(whitelistField.javaFieldName, painlessField);
+            } else if (painlessField.clazz != painlessFieldClass) {
+                throw new IllegalArgumentException("illegal duplicate static fields [" + whitelistField.javaFieldName + "] " +
+                    "found within the struct [" + ownerStruct.name + "] with type [" + whitelistField.painlessFieldTypeName + "]");
+            }
+        } else {
+            MethodHandle javaMethodHandleGetter;
+            MethodHandle javaMethodHandleSetter;
+
+            try {
+                if (Modifier.isStatic(javaField.getModifiers()) == false) {
+                    javaMethodHandleGetter = MethodHandles.publicLookup().unreflectGetter(javaField);
+                    javaMethodHandleSetter = MethodHandles.publicLookup().unreflectSetter(javaField);
+                } else {
+                    javaMethodHandleGetter = null;
+                    javaMethodHandleSetter = null;
+                }
+            } catch (IllegalAccessException exception) {
+                throw new IllegalArgumentException("getter/setter [" + whitelistField.javaFieldName + "]" +
+                    " not found for class [" + ownerStruct.clazz.getName() + "].");
+            }
+
+            PainlessField painlessField = ownerStruct.members.get(whitelistField.javaFieldName);
+
+            if (painlessField == null) {
+                painlessField = fieldCache.computeIfAbsent(
+                    buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()),
+                    key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
+                        ownerStruct, painlessFieldClass, javaField.getModifiers(), javaMethodHandleGetter, javaMethodHandleSetter));
+                ownerStruct.members.put(whitelistField.javaFieldName, painlessField);
+            } else if (painlessField.clazz != painlessFieldClass) {
+                throw new IllegalArgumentException("illegal duplicate member fields [" + whitelistField.javaFieldName + "] " +
+                    "found within the struct [" + ownerStruct.name + "] with type [" + whitelistField.painlessFieldTypeName + "]");
+            }
+        }
+    }
+
+    private void copyStruct(String struct, List<String> children) {
+        final PainlessClass owner = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(struct));
+
+        if (owner == null) {
+            throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
+        }
+
+        for (int count = 0; count < children.size(); ++count) {
+            final PainlessClass child = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(children.get(count)));
+
+            if (child == null) {
+                throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
+                    " not defined for copy to owner struct [" + owner.name + "].");
+            }
+
+            if (!child.clazz.isAssignableFrom(owner.clazz)) {
+                throw new ClassCastException("Child struct [" + child.name + "]" +
+                    " is not a super type of owner struct [" + owner.name + "] in copy.");
+            }
+
+            for (Map.Entry<PainlessMethodKey,PainlessMethod> kvPair : child.methods.entrySet()) {
+                PainlessMethodKey methodKey = kvPair.getKey();
+                PainlessMethod method = kvPair.getValue();
+                if (owner.methods.get(methodKey) == null) {
+                    // TODO: some of these are no longer valid or outright don't work
+                    // TODO: since classes may not come from the Painless classloader
+                    // TODO: and it was dependent on the order of the extends which
+                    // TODO: which no longer exists since this is generated automatically
+                    // sanity check, look for missing covariant/generic override
+                    /*if (owner.clazz.isInterface() && child.clazz == Object.class) {
+                        // ok
+                    } else if (child.clazz == Spliterator.OfPrimitive.class || child.clazz == PrimitiveIterator.class) {
+                        // ok, we rely on generics erasure for these (its guaranteed in the javadocs though!!!!)
+                    } else if (Constants.JRE_IS_MINIMUM_JAVA9 && owner.clazz == LocalDate.class) {
+                        // ok, java 9 added covariant override for LocalDate.getEra() to return IsoEra:
+                        // https://bugs.openjdk.java.net/browse/JDK-8072746
+                    } else {
+                        try {
+                            // TODO: we *have* to remove all these public members and use getter methods to encapsulate!
+                            final Class<?> impl;
+                            final Class<?> arguments[];
+                            if (method.augmentation != null) {
+                                impl = method.augmentation;
+                                arguments = new Class<?>[method.arguments.size() + 1];
+                                arguments[0] = method.owner.clazz;
+                                for (int i = 0; i < method.arguments.size(); i++) {
+                                    arguments[i + 1] = method.arguments.get(i).clazz;
+                                }
+                            } else {
+                                impl = owner.clazz;
+                                arguments = new Class<?>[method.arguments.size()];
+                                for (int i = 0; i < method.arguments.size(); i++) {
+                                    arguments[i] = method.arguments.get(i).clazz;
+                                }
+                            }
+                            java.lang.reflect.Method m = impl.getMethod(method.method.getName(), arguments);
+                            if (m.getReturnType() != method.rtn.clazz) {
+                                throw new IllegalStateException("missing covariant override for: " + m + " in " + owner.name);
+                            }
+                            if (m.isBridge() && !Modifier.isVolatile(method.modifiers)) {
+                                // its a bridge in the destination, but not in the source, but it might still be ok, check generics:
+                                java.lang.reflect.Method source = child.clazz.getMethod(method.method.getName(), arguments);
+                                if (!Arrays.equals(source.getGenericParameterTypes(), source.getParameterTypes())) {
+                                    throw new IllegalStateException("missing generic override for: " + m + " in " + owner.name);
+                                }
+                            }
+                        } catch (ReflectiveOperationException e) {
+                            throw new AssertionError(e);
+                        }
+                    }*/
+                    owner.methods.put(methodKey, method);
+                }
+            }
+
+            for (PainlessField field : child.members.values()) {
+                if (owner.members.get(field.name) == null) {
+                    owner.members.put(field.name,
+                        new PainlessField(field.name, field.javaName, owner, field.clazz, field.modifiers, field.getter, field.setter));
+                }
+            }
+        }
+    }
+
+    /**
+     * Precomputes a more efficient structure for dynamic method/field access.
+     */
+    private void addRuntimeClass(final PainlessClass struct) {
+        // add all getters/setters
+        for (Map.Entry<PainlessMethodKey, PainlessMethod> method : struct.methods.entrySet()) {
+            String name = method.getKey().name;
+            PainlessMethod m = method.getValue();
+
+            if (m.arguments.size() == 0 &&
+                name.startsWith("get") &&
+                name.length() > 3 &&
+                Character.isUpperCase(name.charAt(3))) {
+                StringBuilder newName = new StringBuilder();
+                newName.append(Character.toLowerCase(name.charAt(3)));
+                newName.append(name.substring(4));
+                struct.getters.putIfAbsent(newName.toString(), m.handle);
+            } else if (m.arguments.size() == 0 &&
+                name.startsWith("is") &&
+                name.length() > 2 &&
+                Character.isUpperCase(name.charAt(2))) {
+                StringBuilder newName = new StringBuilder();
+                newName.append(Character.toLowerCase(name.charAt(2)));
+                newName.append(name.substring(3));
+                struct.getters.putIfAbsent(newName.toString(), m.handle);
+            }
+
+            if (m.arguments.size() == 1 &&
+                name.startsWith("set") &&
+                name.length() > 3 &&
+                Character.isUpperCase(name.charAt(3))) {
+                StringBuilder newName = new StringBuilder();
+                newName.append(Character.toLowerCase(name.charAt(3)));
+                newName.append(name.substring(4));
+                struct.setters.putIfAbsent(newName.toString(), m.handle);
+            }
+        }
+
+        // add all members
+        for (Map.Entry<String, PainlessField> member : struct.members.entrySet()) {
+            struct.getters.put(member.getKey(), member.getValue().getter);
+            struct.setters.put(member.getKey(), member.getValue().setter);
+        }
+    }
+
+    /** computes the functional interface method for a class, or returns null */
+    private PainlessMethod computeFunctionalInterfaceMethod(PainlessClass clazz) {
+        if (!clazz.clazz.isInterface()) {
+            return null;
+        }
+        // if its marked with this annotation, we fail if the conditions don't hold (means whitelist bug)
+        // otherwise, this annotation is pretty useless.
+        boolean hasAnnotation = clazz.clazz.isAnnotationPresent(FunctionalInterface.class);
+        List<java.lang.reflect.Method> methods = new ArrayList<>();
+        for (java.lang.reflect.Method m : clazz.clazz.getMethods()) {
+            // default interface methods don't count
+            if (m.isDefault()) {
+                continue;
+            }
+            // static methods don't count
+            if (Modifier.isStatic(m.getModifiers())) {
+                continue;
+            }
+            // if its from Object, it doesn't count
+            try {
+                Object.class.getMethod(m.getName(), m.getParameterTypes());
+                continue;
+            } catch (ReflectiveOperationException e) {
+                // it counts
+            }
+            methods.add(m);
+        }
+        if (methods.size() != 1) {
+            if (hasAnnotation) {
+                throw new IllegalArgumentException("Class: " + clazz.name +
+                    " is marked with FunctionalInterface but doesn't fit the bill: " + methods);
+            }
+            return null;
+        }
+        // inspect the one method found from the reflection API, it should match the whitelist!
+        java.lang.reflect.Method oneMethod = methods.get(0);
+        PainlessMethod painless = clazz.methods.get(new PainlessMethodKey(oneMethod.getName(), oneMethod.getParameterCount()));
+        if (painless == null || painless.method.equals(org.objectweb.asm.commons.Method.getMethod(oneMethod)) == false) {
+            throw new IllegalArgumentException("Class: " + clazz.name + " is functional but the functional " +
+                "method is not whitelisted!");
+        }
+        return painless;
+    }
+
+    public Class<?> getJavaClassFromPainlessType(String painlessType) {
+        Class<?> javaClass = painlessTypesToJavaClasses.get(painlessType);
+
+        if (javaClass != null) {
+            return javaClass;
+        }
+        int arrayDimensions = 0;
+        int arrayIndex = painlessType.indexOf('[');
+
+        if (arrayIndex != -1) {
+            int length = painlessType.length();
+
+            while (arrayIndex < length) {
+                if (painlessType.charAt(arrayIndex) == '[' && ++arrayIndex < length && painlessType.charAt(arrayIndex++) == ']') {
+                    ++arrayDimensions;
+                } else {
+                    throw new IllegalArgumentException("invalid painless type [" + painlessType + "].");
+                }
+            }
+
+            painlessType = painlessType.substring(0, painlessType.indexOf('['));
+            javaClass = painlessTypesToJavaClasses.get(painlessType);
+
+            char braces[] = new char[arrayDimensions];
+            Arrays.fill(braces, '[');
+            String descriptor = new String(braces);
+
+            if (javaClass == boolean.class) {
+                descriptor += "Z";
+            } else if (javaClass == byte.class) {
+                descriptor += "B";
+            } else if (javaClass == short.class) {
+                descriptor += "S";
+            } else if (javaClass == char.class) {
+                descriptor += "C";
+            } else if (javaClass == int.class) {
+                descriptor += "I";
+            } else if (javaClass == long.class) {
+                descriptor += "J";
+            } else if (javaClass == float.class) {
+                descriptor += "F";
+            } else if (javaClass == double.class) {
+                descriptor += "D";
+            } else {
+                descriptor += "L" + javaClass.getName() + ";";
+            }
+
+            try {
+                return Class.forName(descriptor);
+            } catch (ClassNotFoundException cnfe) {
+                throw new IllegalStateException("invalid painless type [" + painlessType + "]", cnfe);
+            }
+        }
+
+        throw new IllegalArgumentException("invalid painless type [" + painlessType + "]");
+    }
+
+    public PainlessLookup build() {
+        return new PainlessLookup(painlessTypesToJavaClasses, javaClassesToPainlessStructs);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/def.java
new file mode 100644
index 00000000000..4336236be3f
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/def.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.lookup;
+
+/** Marker class for def type to be used during type analysis. */
+public final class def {
+
+    private def() {
+
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
index dda246b5f6c..a0a29ed59dd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
@@ -23,7 +23,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.lookup.PainlessCast;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 46fbeefd6f5..422300072dc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index a3e1b4bde6a..c0345b6308c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index c0fccab8e8a..806204d051a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index a7b7a41fe05..8977f4f0ef3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index 518f1953525..820cce685ed 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index 45158aedcf7..b6c7fb80af9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 8e293556eac..3a5102ebdc9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index ec7d0f6d7bb..5b282abdce9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index 12ff4832483..f23ae9f1887 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.PainlessClass;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index 8d27162fc36..78a18b91ab2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessClass;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
index 8e30d434329..ccbc25db4f2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
index 0882f191770..a9021000e2d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
index 41fcf563d24..1c081c9422e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index e7d18ece059..c402d8982d8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index faee2ed74a6..cfc87536b6b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
-import org.elasticsearch.painless.lookup.PainlessLookup.def;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
index 78e5814e963..c0e0bd7ed9d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.spi.Whitelist;
 
 import static java.util.Collections.emptyMap;
@@ -37,7 +38,7 @@ import static org.hamcrest.Matchers.startsWith;
  */
 public class BaseClassTests extends ScriptTestCase {
 
-    private final PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS);
+    private final PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
 
     public abstract static class Gets {
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
index 987eef31eee..7edc90bb0a0 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.script.ScriptException;
 
@@ -36,7 +37,7 @@ import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.not;
 
 public class DebugTests extends ScriptTestCase {
-    private final PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS);
+    private final PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
 
     public void testExplain() {
         // Debug.explain can explain an object
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index 0d5e2748b7b..73adf92779d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.objectweb.asm.util.Textifier;
 
@@ -40,7 +40,7 @@ final class Debugger {
         PrintWriter outputWriter = new PrintWriter(output);
         Textifier textifier = new Textifier();
         try {
-            new Compiler(iface, new PainlessLookup(Whitelist.BASE_WHITELISTS))
+            new Compiler(iface, new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build())
                     .compile("<debugging>", source, settings, textifier);
         } catch (RuntimeException e) {
             textifier.print(outputWriter);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index ab4844dd58b..07f45ff67c0 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -28,11 +28,12 @@ import java.util.Collections;
 import java.util.HashMap;
 
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.test.ESTestCase;
 
 public class DefBootstrapTests extends ESTestCase {
-    private final PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS);
+    private final PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
 
     /** calls toString() on integers, twice */
     public void testOneType() throws Throwable {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index 5177d64cbdb..5e8e6ad47d8 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -20,13 +20,16 @@
 package org.elasticsearch.painless;
 
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessField;
-import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessField;
+import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.spi.Whitelist;
+
 import java.io.IOException;
 import java.io.PrintStream;
 import java.lang.reflect.Modifier;
@@ -42,14 +45,13 @@ import java.util.function.Consumer;
 
 import static java.util.Comparator.comparing;
 import static java.util.stream.Collectors.toList;
-import static org.elasticsearch.painless.spi.Whitelist.BASE_WHITELISTS;
 
 /**
  * Generates an API reference from the method and type whitelists in {@link PainlessLookup}.
  */
 public class PainlessDocGenerator {
 
-    private static final PainlessLookup PAINLESS_LOOKUP = new PainlessLookup(BASE_WHITELISTS);
+    private static final PainlessLookup PAINLESS_LOOKUP = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
     private static final Logger logger = ESLoggerFactory.getLogger(PainlessDocGenerator.class);
     private static final Comparator<PainlessField> FIELD_NAME = comparing(f -> f.name);
     private static final Comparator<PainlessMethod> METHOD_NAME = comparing(m -> m.name);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index 1a4770e560a..eebf1d701ee 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.painless.antlr.Walker;
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptContext;
@@ -91,7 +92,7 @@ public abstract class ScriptTestCase extends ESTestCase {
     public Object exec(String script, Map<String, Object> vars, Map<String,String> compileParams, Scorer scorer, boolean picky) {
         // test for ambiguity errors before running the actual script if picky is true
         if (picky) {
-            PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS);
+            PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
             ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, GenericElasticsearchScript.class);
             CompilerSettings pickySettings = new CompilerSettings();
             pickySettings.setPicky(true);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index 3e9f724743f..86d365e0fcc 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.lookup.PainlessField;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.PainlessClass;
@@ -48,7 +49,7 @@ import static org.elasticsearch.painless.node.SSource.MainMethodReserved;
  * Tests {@link Object#toString} implementations on all extensions of {@link ANode}.
  */
 public class NodeToStringTests extends ESTestCase {
-    private final PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS);
+    private final PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
 
     public void testEAssignment() {
         assertToString(

From ecd05d5be408d5b864189c1c51b9154d627e536b Mon Sep 17 00:00:00 2001
From: DeDe Morton <dede.morton@elastic.co>
Date: Mon, 16 Jul 2018 12:11:24 -0700
Subject: [PATCH 039/260] Use correct formatting for links (#29460)

---
 docs/reference/how-to/recipes.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/how-to/recipes.asciidoc b/docs/reference/how-to/recipes.asciidoc
index e798f8819d0..451e192ad6a 100644
--- a/docs/reference/how-to/recipes.asciidoc
+++ b/docs/reference/how-to/recipes.asciidoc
@@ -3,8 +3,8 @@
 
 This section includes a few recipes to help with common problems:
 
-* mixing-exact-search-with-stemming
-* consistent-scoring
+* <<mixing-exact-search-with-stemming>>
+* <<consistent-scoring>>
 
 include::recipes/stemming.asciidoc[]
 include::recipes/scoring.asciidoc[]

From 637cac90614018d6e46c781547ddac2000d71459 Mon Sep 17 00:00:00 2001
From: Michael Basnight <mbasnight@gmail.com>
Date: Mon, 16 Jul 2018 14:20:16 -0500
Subject: [PATCH 040/260] Watcher: Store username on watch execution (#31873)

There is currently no way to see what user executed a watch. This commit
adds the decrypted username to each execution in the watch history, in a
new field "user".

Closes #31772
---
 .../rest-api/watcher/execute-watch.asciidoc   |  4 +-
 .../core/security/authc/Authentication.java   | 10 +++-
 .../execution/WatchExecutionContext.java      | 25 +++++++++
 .../core/watcher/history/WatchRecord.java     | 17 ++++--
 .../WatcherIndexTemplateRegistryField.java    |  3 +-
 .../src/main/resources/watch-history.json     |  3 +
 .../execution/ExecutionServiceTests.java      | 31 ++++++++++
 .../roles.yml                                 |  1 +
 .../20_test_run_as_execute_watch.yml          | 56 +++++++++++++++++++
 9 files changed, 140 insertions(+), 10 deletions(-)

diff --git a/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc b/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc
index 91cd89bca6d..ec2c60c543b 100644
--- a/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc
+++ b/x-pack/docs/en/rest-api/watcher/execute-watch.asciidoc
@@ -263,7 +263,8 @@ This is an example of the output:
           "type": "index"
         }
       ]
-    }
+    },
+    "user": "test_admin" <4>
   }
 }
 --------------------------------------------------
@@ -281,6 +282,7 @@ This is an example of the output:
 <1> The id of the watch record as it would be stored in the `.watcher-history` index.
 <2> The watch record document as it would be stored in the `.watcher-history` index.
 <3> The watch execution results.
+<4> The user used to execute the watch.
 
 You can set a different execution mode for every action by associating the mode
 name with the action id:
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
index 2a2fdd95d61..161d9d44999 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
@@ -88,13 +88,17 @@ public class Authentication {
             throws IOException, IllegalArgumentException {
         assert ctx.getTransient(AuthenticationField.AUTHENTICATION_KEY) == null;
 
+        Authentication authentication = decode(header);
+        ctx.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication);
+        return authentication;
+    }
+
+    public static Authentication decode(String header) throws IOException {
         byte[] bytes = Base64.getDecoder().decode(header);
         StreamInput input = StreamInput.wrap(bytes);
         Version version = Version.readVersion(input);
         input.setVersion(version);
-        Authentication authentication = new Authentication(input);
-        ctx.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication);
-        return authentication;
+        return new Authentication(input);
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionContext.java
index 62216ff681e..dbbff33dcef 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionContext.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/execution/WatchExecutionContext.java
@@ -8,6 +8,8 @@ package org.elasticsearch.xpack.core.watcher.execution;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.watcher.actions.ActionWrapperResult;
 import org.elasticsearch.xpack.core.watcher.condition.Condition;
 import org.elasticsearch.xpack.core.watcher.history.WatchRecord;
@@ -18,6 +20,7 @@ import org.elasticsearch.xpack.core.watcher.watch.Payload;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.joda.time.DateTime;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -43,6 +46,7 @@ public abstract class WatchExecutionContext {
     private Transform.Result transformResult;
     private ConcurrentMap<String, ActionWrapperResult> actionsResults = ConcurrentCollections.newConcurrentMap();
     private String nodeId;
+    private String user;
 
     public WatchExecutionContext(String watchId, DateTime executionTime, TriggerEvent triggerEvent, TimeValue defaultThrottlePeriod) {
         this.id = new Wid(watchId, executionTime);
@@ -85,6 +89,7 @@ public abstract class WatchExecutionContext {
     public final void ensureWatchExists(CheckedSupplier<Watch, Exception> supplier) throws Exception {
         if (watch == null) {
             watch = supplier.get();
+            user = WatchExecutionContext.getUsernameFromWatch(watch);
         }
     }
 
@@ -137,6 +142,11 @@ public abstract class WatchExecutionContext {
         return nodeId;
     }
 
+    /**
+     * @return The user that executes the watch, which will be stored in the watch history
+     */
+    public String getUser() { return user; }
+
     public void start() {
         assert phase == ExecutionPhase.AWAITS_EXECUTION;
         relativeStartTime = System.nanoTime();
@@ -243,4 +253,19 @@ public abstract class WatchExecutionContext {
     public WatchExecutionSnapshot createSnapshot(Thread executionThread) {
         return new WatchExecutionSnapshot(this, executionThread.getStackTrace());
     }
+
+    /**
+     * Given a watch, this extracts and decodes the relevant auth header and returns the principal of the user that is
+     * executing the watch.
+     */
+    public static String getUsernameFromWatch(Watch watch) throws IOException {
+        if (watch != null && watch.status() != null && watch.status().getHeaders() != null) {
+            String header = watch.status().getHeaders().get(AuthenticationField.AUTHENTICATION_KEY);
+            if (header != null) {
+                Authentication auth = Authentication.decode(header);
+                return auth.getUser().principal();
+            }
+        }
+        return null;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/WatchRecord.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/WatchRecord.java
index 74e7b2115fa..2b28c2f15c9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/WatchRecord.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/WatchRecord.java
@@ -43,12 +43,14 @@ public abstract class WatchRecord implements ToXContentObject {
     private static final ParseField METADATA = new ParseField("metadata");
     private static final ParseField EXECUTION_RESULT = new ParseField("result");
     private static final ParseField EXCEPTION = new ParseField("exception");
+    private static final ParseField USER = new ParseField("user");
 
     protected final Wid id;
     protected final Watch watch;
     private final String nodeId;
     protected final TriggerEvent triggerEvent;
     protected final ExecutionState state;
+    private final String user;
 
     // only emitted to xcontent in "debug" mode
     protected final Map<String, Object> vars;
@@ -60,7 +62,7 @@ public abstract class WatchRecord implements ToXContentObject {
 
     private WatchRecord(Wid id, TriggerEvent triggerEvent, ExecutionState state, Map<String, Object> vars, ExecutableInput input,
                         ExecutableCondition condition, Map<String, Object> metadata, Watch watch, WatchExecutionResult executionResult,
-                        String nodeId) {
+                        String nodeId, String user) {
         this.id = id;
         this.triggerEvent = triggerEvent;
         this.state = state;
@@ -71,15 +73,16 @@ public abstract class WatchRecord implements ToXContentObject {
         this.executionResult = executionResult;
         this.watch = watch;
         this.nodeId = nodeId;
+        this.user = user;
     }
 
     private WatchRecord(Wid id, TriggerEvent triggerEvent, ExecutionState state, String nodeId) {
-        this(id, triggerEvent, state, Collections.emptyMap(), null, null, null, null, null, nodeId);
+        this(id, triggerEvent, state, Collections.emptyMap(), null, null, null, null, null, nodeId, null);
     }
 
     private WatchRecord(WatchRecord record, ExecutionState state) {
         this(record.id, record.triggerEvent, state, record.vars, record.input, record.condition, record.metadata, record.watch,
-                record.executionResult, record.nodeId);
+                record.executionResult, record.nodeId, record.user);
     }
 
     private WatchRecord(WatchExecutionContext context, ExecutionState state) {
@@ -88,12 +91,13 @@ public abstract class WatchRecord implements ToXContentObject {
                 context.watch() != null ? context.watch().condition() : null,
                 context.watch() != null ? context.watch().metadata() : null,
                 context.watch(),
-                null, context.getNodeId());
+                null, context.getNodeId(), context.getUser());
     }
 
     private WatchRecord(WatchExecutionContext context, WatchExecutionResult executionResult) {
         this(context.id(), context.triggerEvent(), getState(executionResult), context.vars(), context.watch().input(),
-                context.watch().condition(), context.watch().metadata(), context.watch(), executionResult, context.getNodeId());
+                context.watch().condition(), context.watch().metadata(), context.watch(), executionResult, context.getNodeId(),
+                context.getUser());
     }
 
     public static ExecutionState getState(WatchExecutionResult executionResult) {
@@ -152,6 +156,9 @@ public abstract class WatchRecord implements ToXContentObject {
         builder.field(NODE.getPreferredName(), nodeId);
         builder.field(STATE.getPreferredName(), state.id());
 
+        if (user != null) {
+            builder.field(USER.getPreferredName(), user);
+        }
         if (watch != null && watch.status() != null) {
             builder.field(STATUS.getPreferredName(), watch.status(), params);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java
index 25e2c928d9a..b42506b81b3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java
@@ -13,8 +13,9 @@ public final class WatcherIndexTemplateRegistryField {
     // version 6: upgrade to ES 6, removal of _status field
     // version 7: add full exception stack traces for better debugging
     // version 8: fix slack attachment property not to be dynamic, causing field type issues
+    // version 9: add a user field defining which user executed the watch
     // Note: if you change this, also inform the kibana team around the watcher-ui
-    public static final String INDEX_TEMPLATE_VERSION = "8";
+    public static final String INDEX_TEMPLATE_VERSION = "9";
     public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";
     public static final String WATCHES_TEMPLATE_NAME = ".watches";
diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json
index 86a967fc14f..9a4a96409b0 100644
--- a/x-pack/plugin/core/src/main/resources/watch-history.json
+++ b/x-pack/plugin/core/src/main/resources/watch-history.json
@@ -120,6 +120,9 @@
         "messages": {
           "type": "text"
         },
+        "user": {
+          "type": "text"
+        },
         "exception" : {
           "type" : "object",
           "enabled" : false
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
index 73f0e820720..d3f46d3d452 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
@@ -31,6 +31,9 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
+import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.watcher.actions.Action;
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.core.watcher.actions.ActionWrapper;
@@ -85,6 +88,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
@@ -1072,6 +1076,33 @@ public class ExecutionServiceTests extends ESTestCase {
         assertThat(watchRecord.state(), is(ExecutionState.EXECUTED));
     }
 
+    public void testLoadingWatchExecutionUser() throws Exception {
+        DateTime now = now(UTC);
+        Watch watch = mock(Watch.class);
+        WatchStatus status = mock(WatchStatus.class);
+        ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now);
+
+        // Should be null
+        TriggeredExecutionContext context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5));
+        context.ensureWatchExists(() -> watch);
+        assertNull(context.getUser());
+
+        // Should still be null, header is not yet set
+        when(watch.status()).thenReturn(status);
+        context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5));
+        context.ensureWatchExists(() -> watch);
+        assertNull(context.getUser());
+
+        Authentication authentication = new Authentication(new User("joe", "admin"),
+            new Authentication.RealmRef("native_realm", "native", "node1"), null);
+
+        // Should no longer be null now that the proper header is set
+        when(status.getHeaders()).thenReturn(Collections.singletonMap(AuthenticationField.AUTHENTICATION_KEY, authentication.encode()));
+        context = new TriggeredExecutionContext(watch.id(), now, event, timeValueSeconds(5));
+        context.ensureWatchExists(() -> watch);
+        assertThat(context.getUser(), equalTo("joe"));
+    }
+
     private WatchExecutionContext createMockWatchExecutionContext(String watchId, DateTime executionTime) {
         WatchExecutionContext ctx = mock(WatchExecutionContext.class);
         when(ctx.id()).thenReturn(new Wid(watchId, executionTime));
diff --git a/x-pack/qa/smoke-test-watcher-with-security/roles.yml b/x-pack/qa/smoke-test-watcher-with-security/roles.yml
index bebfa883fcb..b52fe6c5c59 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/roles.yml
+++ b/x-pack/qa/smoke-test-watcher-with-security/roles.yml
@@ -21,6 +21,7 @@ watcher_manager:
   run_as:
     - powerless_user
     - watcher_manager
+    - x_pack_rest_user
 
 watcher_monitor:
   cluster:
diff --git a/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml b/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml
index 9bc7724b2c0..7a0634f5187 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml
+++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml
@@ -74,10 +74,63 @@ teardown:
         id: "my_watch"
   - match: { watch_record.watch_id: "my_watch" }
   - match: { watch_record.state: "executed" }
+  - match: { watch_record.user: "watcher_manager" }
 
 
 
 
+---
+"Test watch is runas user properly recorded":
+  - do:
+      xpack.watcher.put_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "trigger": {
+              "schedule" : { "cron" : "0 0 0 1 * ? 2099" }
+            },
+            "input": {
+              "search" : {
+                "request" : {
+                  "indices" : [ "my_test_index" ],
+                  "body" :{
+                    "query" : { "match_all": {} }
+                  }
+                }
+              }
+            },
+            "condition" : {
+              "compare" : {
+                "ctx.payload.hits.total" : {
+                  "gte" : 1
+                }
+              }
+            },
+            "actions": {
+              "logging": {
+                "logging": {
+                  "text": "Successfully ran my_watch to test for search input"
+                }
+              }
+            }
+          }
+  - match: { _id: "my_watch" }
+
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { _id: "my_watch" }
+  - is_false: watch.status.headers
+
+  - do:
+      headers: { es-security-runas-user: x_pack_rest_user }
+      xpack.watcher.execute_watch:
+        id: "my_watch"
+  - match: { watch_record.watch_id: "my_watch" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.user: "x_pack_rest_user" }
+
+
 ---
 "Test watch search input does not work against index user is not allowed to read":
 
@@ -130,6 +183,7 @@ teardown:
   - match: { watch_record.watch_id: "my_watch" }
   # because we are not allowed to read the index, there wont be any data
   - match: { watch_record.state: "execution_not_needed" }
+  - match: { watch_record.user: "watcher_manager" }
 
 
 ---
@@ -272,6 +326,7 @@ teardown:
         id: "my_watch"
   - match: { watch_record.watch_id: "my_watch" }
   - match: { watch_record.state: "executed" }
+  - match: { watch_record.user: "watcher_manager" }
 
   - do:
       get:
@@ -320,6 +375,7 @@ teardown:
         id: "my_watch"
   - match: { watch_record.watch_id: "my_watch" }
   - match: { watch_record.state: "executed" }
+  - match: { watch_record.user: "watcher_manager" }
 
   - do:
       get:

From cccc3f7a641f4e9cbe73adf968ffe1419c715a3f Mon Sep 17 00:00:00 2001
From: debadair <debadair@elastic.co>
Date: Mon, 16 Jul 2018 12:47:57 -0700
Subject: [PATCH 041/260] Tweaked Elasticsearch Service links for SEO

---
 docs/reference/setup/install.asciidoc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc
index 4433ffb8c38..c0ebfb60fa7 100644
--- a/docs/reference/setup/install.asciidoc
+++ b/docs/reference/setup/install.asciidoc
@@ -3,10 +3,12 @@
 
 [float]
 === Hosted Elasticsearch
-Elasticsearch can be run on your own hardware or using our hosted
-Elasticsearch Service on https://www.elastic.co/cloud[Elastic Cloud], which is
-available on AWS and GCP. You can
-https://www.elastic.co/cloud/elasticsearch-service/signup[try out the hosted service] for free.
+
+You can run Elasticsearch on your own hardware, or use our
+https://www.elastic.co/cloud/elasticsearch-service[hosted Elasticsearch Service]
+on Elastic Cloud. The Elasticsearch Service is available on both AWS and GCP.
+https://www.elastic.co/cloud/elasticsearch-service/signup[Try out the
+Elasticsearch Service for free].
 
 [float]
 === Installing Elasticsearch Yourself

From ff8260c234a6f522941528e82a11fcde6f181ce8 Mon Sep 17 00:00:00 2001
From: debadair <debadair@elastic.co>
Date: Mon, 16 Jul 2018 12:58:08 -0700
Subject: [PATCH 042/260] Tweaked Elasticsearch Service links for SEO

---
 docs/reference/getting-started.asciidoc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index ff00c310a43..e2dae285688 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -106,10 +106,11 @@ With that out of the way, let's get started with the fun part...
 
 [TIP]
 ==============
-You can skip installation completely by using our hosted
-Elasticsearch Service on https://www.elastic.co/cloud[Elastic Cloud], which is
-available on AWS and GCP. You can
-https://www.elastic.co/cloud/elasticsearch-service/signup[try out the hosted service] for free.
+You can skip having to install Elasticsearch by using our
+https://www.elastic.co/cloud/elasticsearch-service[hosted Elasticsearch Service]
+on Elastic Cloud. The Elasticsearch Service is available on both AWS and GCP.
+https://www.elastic.co/cloud/elasticsearch-service/signup[Try out the
+Elasticsearch Service for free].
 ==============
 
 Elasticsearch requires at least Java 8. Specifically as of this writing, it is recommended that you use the Oracle JDK version {jdk}. Java installation varies from platform to platform so we won't go into those details here. Oracle's recommended installation documentation can be found on http://docs.oracle.com/javase/8/docs/technotes/guides/install/install_overview.html[Oracle's website]. Suffice to say, before you install Elasticsearch, please check your Java version first by running (and then install/upgrade accordingly if needed):

From 69c85331b0f574323994dc1ccb4f70ab0e122a4c Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 16 Jul 2018 13:02:53 -0700
Subject: [PATCH 043/260] [test] turn on host io cache for opensuse (#32053)

The hope is that this will resolve the problems with very slow io we're
seeing on this box in #30295
---
 Vagrantfile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Vagrantfile b/Vagrantfile
index 7322399fed5..de344e18183 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -115,6 +115,11 @@ Vagrant.configure(2) do |config|
   'opensuse-42'.tap do |box|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/opensuse-42-x86_64'
+
+      # https://github.com/elastic/elasticsearch/issues/30295
+      config.vm.provider 'virtualbox' do |vbox|
+        vbox.customize ['storagectl', :id, '--name', 'SATA Controller', '--hostiocache', 'on']
+      end
       suse_common config, box
     end
   end

From 5b65e2ccbf66370524b66a11305ebd4555cad27b Mon Sep 17 00:00:00 2001
From: ahmedakef <aemed.akef.1@gmail.com>
Date: Mon, 16 Jul 2018 22:00:57 +0200
Subject: [PATCH 044/260] DOCS: put LIMIT 10 to the SQL query (#32065)

Provides a more precise equivalent SQL query for the aggregation example in the getting started guide.
---
 docs/reference/getting-started.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index e2dae285688..b89021e1cfe 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -1122,7 +1122,7 @@ In SQL, the above aggregation is similar in concept to:
 
 [source,sh]
 --------------------------------------------------
-SELECT state, COUNT(*) FROM bank GROUP BY state ORDER BY COUNT(*) DESC
+SELECT state, COUNT(*) FROM bank GROUP BY state ORDER BY COUNT(*) DESC LIMIT 10;
 --------------------------------------------------
 
 And the response (partially shown):

From 20ea72e2f26ed0c4d21b6865820298a26091daa8 Mon Sep 17 00:00:00 2001
From: Costin Leau <costin@users.noreply.github.com>
Date: Mon, 16 Jul 2018 23:42:44 +0300
Subject: [PATCH 045/260] SQL: allow LEFT and RIGHT as function names (#32066)

Due to the way ANTLR works, any declared tokens need to be accounted for
manually inside function names (otherwise a different rule gets applied).

Fix #32046
---
 x-pack/plugin/sql/src/main/antlr/SqlBase.g4   |    7 +-
 .../xpack/sql/parser/ExpressionBuilder.java   |    3 +-
 .../xpack/sql/parser/SqlBaseBaseListener.java |   12 +
 .../xpack/sql/parser/SqlBaseBaseVisitor.java  |    7 +
 .../xpack/sql/parser/SqlBaseListener.java     |   10 +
 .../xpack/sql/parser/SqlBaseParser.java       | 1706 +++++++++--------
 .../xpack/sql/parser/SqlBaseVisitor.java      |    6 +
 .../xpack/sql/parser/ExpressionTests.java     |    8 +
 .../xpack/sql/parser/SqlParserTests.java      |    6 +
 9 files changed, 961 insertions(+), 804 deletions(-)

diff --git a/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 b/x-pack/plugin/sql/src/main/antlr/SqlBase.g4
index 2c3288babd6..490a5486b42 100644
--- a/x-pack/plugin/sql/src/main/antlr/SqlBase.g4
+++ b/x-pack/plugin/sql/src/main/antlr/SqlBase.g4
@@ -243,7 +243,12 @@ functionExpression
     ;
     
 functionTemplate
-    : identifier '(' (setQuantifier? expression (',' expression)*)? ')'
+    : functionName '(' (setQuantifier? expression (',' expression)*)? ')'
+    ;
+functionName
+    : LEFT 
+    | RIGHT 
+    | identifier
     ;
     
 constant
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
index 66ec98ea53c..e202803b261 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
@@ -396,8 +396,7 @@ abstract class ExpressionBuilder extends IdentifierBuilder {
     @Override
     public Function visitFunctionExpression(FunctionExpressionContext ctx) {
         FunctionTemplateContext template = ctx.functionTemplate();
-
-        String name = visitIdentifier(template.identifier());
+        String name = template.functionName().getText();
         boolean isDistinct = template.setQuantifier() != null && template.setQuantifier().DISTINCT() != null;
         UnresolvedFunction.ResolutionType resolutionType =
                 isDistinct ? UnresolvedFunction.ResolutionType.DISTINCT : UnresolvedFunction.ResolutionType.STANDARD;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java
index b353bcf6521..72c417992e3 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java
@@ -803,6 +803,18 @@ class SqlBaseBaseListener implements SqlBaseListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterFunctionName(SqlBaseParser.FunctionNameContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitFunctionName(SqlBaseParser.FunctionNameContext ctx) { }
   /**
    * {@inheritDoc}
    *
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java
index d40ae6daa6e..fd35ec421f6 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java
@@ -473,6 +473,13 @@ class SqlBaseBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements SqlBa
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitFunctionName(SqlBaseParser.FunctionNameContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java
index 70fe777384c..18b2a21c02e 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java
@@ -745,6 +745,16 @@ interface SqlBaseListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx);
+  /**
+   * Enter a parse tree produced by {@link SqlBaseParser#functionName}.
+   * @param ctx the parse tree
+   */
+  void enterFunctionName(SqlBaseParser.FunctionNameContext ctx);
+  /**
+   * Exit a parse tree produced by {@link SqlBaseParser#functionName}.
+   * @param ctx the parse tree
+   */
+  void exitFunctionName(SqlBaseParser.FunctionNameContext ctx);
   /**
    * Enter a parse tree produced by the {@code nullLiteral}
    * labeled alternative in {@link SqlBaseParser#constant}.
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java
index 32a1b062fee..c91758dadbc 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java
@@ -44,10 +44,11 @@ class SqlBaseParser extends Parser {
     RULE_predicate = 24, RULE_pattern = 25, RULE_patternEscape = 26, RULE_valueExpression = 27, 
     RULE_primaryExpression = 28, RULE_castExpression = 29, RULE_castTemplate = 30, 
     RULE_extractExpression = 31, RULE_extractTemplate = 32, RULE_functionExpression = 33, 
-    RULE_functionTemplate = 34, RULE_constant = 35, RULE_comparisonOperator = 36, 
-    RULE_booleanValue = 37, RULE_dataType = 38, RULE_qualifiedName = 39, RULE_identifier = 40, 
-    RULE_tableIdentifier = 41, RULE_quoteIdentifier = 42, RULE_unquoteIdentifier = 43, 
-    RULE_number = 44, RULE_string = 45, RULE_nonReserved = 46;
+    RULE_functionTemplate = 34, RULE_functionName = 35, RULE_constant = 36, 
+    RULE_comparisonOperator = 37, RULE_booleanValue = 38, RULE_dataType = 39, 
+    RULE_qualifiedName = 40, RULE_identifier = 41, RULE_tableIdentifier = 42, 
+    RULE_quoteIdentifier = 43, RULE_unquoteIdentifier = 44, RULE_number = 45, 
+    RULE_string = 46, RULE_nonReserved = 47;
   public static final String[] ruleNames = {
     "singleStatement", "singleExpression", "statement", "query", "queryNoWith", 
     "limitClause", "queryTerm", "orderBy", "querySpecification", "fromClause", 
@@ -56,9 +57,9 @@ class SqlBaseParser extends Parser {
     "relationPrimary", "expression", "booleanExpression", "predicated", "predicate", 
     "pattern", "patternEscape", "valueExpression", "primaryExpression", "castExpression", 
     "castTemplate", "extractExpression", "extractTemplate", "functionExpression", 
-    "functionTemplate", "constant", "comparisonOperator", "booleanValue", 
-    "dataType", "qualifiedName", "identifier", "tableIdentifier", "quoteIdentifier", 
-    "unquoteIdentifier", "number", "string", "nonReserved"
+    "functionTemplate", "functionName", "constant", "comparisonOperator", 
+    "booleanValue", "dataType", "qualifiedName", "identifier", "tableIdentifier", 
+    "quoteIdentifier", "unquoteIdentifier", "number", "string", "nonReserved"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -172,9 +173,9 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(94);
+      setState(96);
       statement();
-      setState(95);
+      setState(97);
       match(EOF);
       }
     }
@@ -219,9 +220,9 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(97);
+      setState(99);
       expression();
-      setState(98);
+      setState(100);
       match(EOF);
       }
     }
@@ -607,14 +608,14 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 4, RULE_statement);
     int _la;
     try {
-      setState(209);
+      setState(211);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
       case 1:
         _localctx = new StatementDefaultContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(100);
+        setState(102);
         query();
         }
         break;
@@ -622,27 +623,27 @@ class SqlBaseParser extends Parser {
         _localctx = new ExplainContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(101);
+        setState(103);
         match(EXPLAIN);
-        setState(115);
+        setState(117);
         _errHandler.sync(this);
         switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) {
         case 1:
           {
-          setState(102);
+          setState(104);
           match(T__0);
-          setState(111);
+          setState(113);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (((((_la - 28)) & ~0x3f) == 0 && ((1L << (_la - 28)) & ((1L << (FORMAT - 28)) | (1L << (PLAN - 28)) | (1L << (VERIFY - 28)))) != 0)) {
             {
-            setState(109);
+            setState(111);
             switch (_input.LA(1)) {
             case PLAN:
               {
-              setState(103);
+              setState(105);
               match(PLAN);
-              setState(104);
+              setState(106);
               ((ExplainContext)_localctx).type = _input.LT(1);
               _la = _input.LA(1);
               if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ALL) | (1L << ANALYZED) | (1L << EXECUTABLE) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED))) != 0)) ) {
@@ -654,9 +655,9 @@ class SqlBaseParser extends Parser {
               break;
             case FORMAT:
               {
-              setState(105);
+              setState(107);
               match(FORMAT);
-              setState(106);
+              setState(108);
               ((ExplainContext)_localctx).format = _input.LT(1);
               _la = _input.LA(1);
               if ( !(_la==GRAPHVIZ || _la==TEXT) ) {
@@ -668,9 +669,9 @@ class SqlBaseParser extends Parser {
               break;
             case VERIFY:
               {
-              setState(107);
+              setState(109);
               match(VERIFY);
-              setState(108);
+              setState(110);
               ((ExplainContext)_localctx).verify = booleanValue();
               }
               break;
@@ -678,16 +679,16 @@ class SqlBaseParser extends Parser {
               throw new NoViableAltException(this);
             }
             }
-            setState(113);
+            setState(115);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
-          setState(114);
+          setState(116);
           match(T__1);
           }
           break;
         }
-        setState(117);
+        setState(119);
         statement();
         }
         break;
@@ -695,27 +696,27 @@ class SqlBaseParser extends Parser {
         _localctx = new DebugContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(118);
+        setState(120);
         match(DEBUG);
-        setState(130);
+        setState(132);
         _errHandler.sync(this);
         switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
         case 1:
           {
-          setState(119);
+          setState(121);
           match(T__0);
-          setState(126);
+          setState(128);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==FORMAT || _la==PLAN) {
             {
-            setState(124);
+            setState(126);
             switch (_input.LA(1)) {
             case PLAN:
               {
-              setState(120);
+              setState(122);
               match(PLAN);
-              setState(121);
+              setState(123);
               ((DebugContext)_localctx).type = _input.LT(1);
               _la = _input.LA(1);
               if ( !(_la==ANALYZED || _la==OPTIMIZED) ) {
@@ -727,9 +728,9 @@ class SqlBaseParser extends Parser {
               break;
             case FORMAT:
               {
-              setState(122);
+              setState(124);
               match(FORMAT);
-              setState(123);
+              setState(125);
               ((DebugContext)_localctx).format = _input.LT(1);
               _la = _input.LA(1);
               if ( !(_la==GRAPHVIZ || _la==TEXT) ) {
@@ -743,16 +744,16 @@ class SqlBaseParser extends Parser {
               throw new NoViableAltException(this);
             }
             }
-            setState(128);
+            setState(130);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
-          setState(129);
+          setState(131);
           match(T__1);
           }
           break;
         }
-        setState(132);
+        setState(134);
         statement();
         }
         break;
@@ -760,24 +761,24 @@ class SqlBaseParser extends Parser {
         _localctx = new ShowTablesContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(133);
+        setState(135);
         match(SHOW);
-        setState(134);
+        setState(136);
         match(TABLES);
-        setState(139);
+        setState(141);
         _la = _input.LA(1);
         if (((((_la - 40)) & ~0x3f) == 0 && ((1L << (_la - 40)) & ((1L << (LIKE - 40)) | (1L << (PARAM - 40)) | (1L << (STRING - 40)))) != 0)) {
           {
-          setState(136);
+          setState(138);
           _la = _input.LA(1);
           if (_la==LIKE) {
             {
-            setState(135);
+            setState(137);
             match(LIKE);
             }
           }
 
-          setState(138);
+          setState(140);
           pattern();
           }
         }
@@ -788,28 +789,13 @@ class SqlBaseParser extends Parser {
         _localctx = new ShowColumnsContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(141);
-        match(SHOW);
-        setState(142);
-        match(COLUMNS);
         setState(143);
-        _la = _input.LA(1);
-        if ( !(_la==FROM || _la==IN) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
+        match(SHOW);
         setState(144);
-        tableIdentifier();
-        }
-        break;
-      case 6:
-        _localctx = new ShowColumnsContext(_localctx);
-        enterOuterAlt(_localctx, 6);
-        {
+        match(COLUMNS);
         setState(145);
         _la = _input.LA(1);
-        if ( !(_la==DESC || _la==DESCRIBE) ) {
+        if ( !(_la==FROM || _la==IN) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -818,28 +804,43 @@ class SqlBaseParser extends Parser {
         tableIdentifier();
         }
         break;
+      case 6:
+        _localctx = new ShowColumnsContext(_localctx);
+        enterOuterAlt(_localctx, 6);
+        {
+        setState(147);
+        _la = _input.LA(1);
+        if ( !(_la==DESC || _la==DESCRIBE) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
+        setState(148);
+        tableIdentifier();
+        }
+        break;
       case 7:
         _localctx = new ShowFunctionsContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(147);
+        setState(149);
         match(SHOW);
-        setState(148);
+        setState(150);
         match(FUNCTIONS);
-        setState(153);
+        setState(155);
         _la = _input.LA(1);
         if (((((_la - 40)) & ~0x3f) == 0 && ((1L << (_la - 40)) & ((1L << (LIKE - 40)) | (1L << (PARAM - 40)) | (1L << (STRING - 40)))) != 0)) {
           {
-          setState(150);
+          setState(152);
           _la = _input.LA(1);
           if (_la==LIKE) {
             {
-            setState(149);
+            setState(151);
             match(LIKE);
             }
           }
 
-          setState(152);
+          setState(154);
           pattern();
           }
         }
@@ -850,9 +851,9 @@ class SqlBaseParser extends Parser {
         _localctx = new ShowSchemasContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(155);
+        setState(157);
         match(SHOW);
-        setState(156);
+        setState(158);
         match(SCHEMAS);
         }
         break;
@@ -860,9 +861,9 @@ class SqlBaseParser extends Parser {
         _localctx = new SysCatalogsContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(157);
+        setState(159);
         match(SYS);
-        setState(158);
+        setState(160);
         match(CATALOGS);
         }
         break;
@@ -870,69 +871,69 @@ class SqlBaseParser extends Parser {
         _localctx = new SysTablesContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(159);
+        setState(161);
         match(SYS);
-        setState(160);
+        setState(162);
         match(TABLES);
-        setState(166);
+        setState(168);
         _la = _input.LA(1);
         if (_la==CATALOG) {
           {
-          setState(161);
-          match(CATALOG);
           setState(163);
+          match(CATALOG);
+          setState(165);
           _la = _input.LA(1);
           if (_la==LIKE) {
             {
-            setState(162);
+            setState(164);
             match(LIKE);
             }
           }
 
-          setState(165);
+          setState(167);
           ((SysTablesContext)_localctx).clusterPattern = pattern();
           }
         }
 
-        setState(172);
+        setState(174);
         _la = _input.LA(1);
         if (((((_la - 40)) & ~0x3f) == 0 && ((1L << (_la - 40)) & ((1L << (LIKE - 40)) | (1L << (PARAM - 40)) | (1L << (STRING - 40)))) != 0)) {
           {
-          setState(169);
+          setState(171);
           _la = _input.LA(1);
           if (_la==LIKE) {
             {
-            setState(168);
+            setState(170);
             match(LIKE);
             }
           }
 
-          setState(171);
+          setState(173);
           ((SysTablesContext)_localctx).tablePattern = pattern();
           }
         }
 
-        setState(183);
+        setState(185);
         _la = _input.LA(1);
         if (_la==TYPE) {
           {
-          setState(174);
+          setState(176);
           match(TYPE);
-          setState(175);
+          setState(177);
           string();
-          setState(180);
+          setState(182);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==T__2) {
             {
             {
-            setState(176);
+            setState(178);
             match(T__2);
-            setState(177);
+            setState(179);
             string();
             }
             }
-            setState(182);
+            setState(184);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
@@ -945,55 +946,55 @@ class SqlBaseParser extends Parser {
         _localctx = new SysColumnsContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(185);
+        setState(187);
         match(SYS);
-        setState(186);
+        setState(188);
         match(COLUMNS);
-        setState(189);
+        setState(191);
         _la = _input.LA(1);
         if (_la==CATALOG) {
           {
-          setState(187);
+          setState(189);
           match(CATALOG);
-          setState(188);
+          setState(190);
           ((SysColumnsContext)_localctx).cluster = string();
           }
         }
 
-        setState(196);
+        setState(198);
         _la = _input.LA(1);
         if (_la==TABLE) {
           {
-          setState(191);
-          match(TABLE);
           setState(193);
+          match(TABLE);
+          setState(195);
           _la = _input.LA(1);
           if (_la==LIKE) {
             {
-            setState(192);
+            setState(194);
             match(LIKE);
             }
           }
 
-          setState(195);
+          setState(197);
           ((SysColumnsContext)_localctx).indexPattern = pattern();
           }
         }
 
-        setState(202);
+        setState(204);
         _la = _input.LA(1);
         if (((((_la - 40)) & ~0x3f) == 0 && ((1L << (_la - 40)) & ((1L << (LIKE - 40)) | (1L << (PARAM - 40)) | (1L << (STRING - 40)))) != 0)) {
           {
-          setState(199);
+          setState(201);
           _la = _input.LA(1);
           if (_la==LIKE) {
             {
-            setState(198);
+            setState(200);
             match(LIKE);
             }
           }
 
-          setState(201);
+          setState(203);
           ((SysColumnsContext)_localctx).columnPattern = pattern();
           }
         }
@@ -1004,9 +1005,9 @@ class SqlBaseParser extends Parser {
         _localctx = new SysTypesContext(_localctx);
         enterOuterAlt(_localctx, 12);
         {
-        setState(204);
+        setState(206);
         match(SYS);
-        setState(205);
+        setState(207);
         match(TYPES);
         }
         break;
@@ -1014,11 +1015,11 @@ class SqlBaseParser extends Parser {
         _localctx = new SysTableTypesContext(_localctx);
         enterOuterAlt(_localctx, 13);
         {
-        setState(206);
-        match(SYS);
-        setState(207);
-        match(TABLE);
         setState(208);
+        match(SYS);
+        setState(209);
+        match(TABLE);
+        setState(210);
         match(TYPES);
         }
         break;
@@ -1072,34 +1073,34 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(220);
+      setState(222);
       _la = _input.LA(1);
       if (_la==WITH) {
         {
-        setState(211);
+        setState(213);
         match(WITH);
-        setState(212);
+        setState(214);
         namedQuery();
-        setState(217);
+        setState(219);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==T__2) {
           {
           {
-          setState(213);
+          setState(215);
           match(T__2);
-          setState(214);
+          setState(216);
           namedQuery();
           }
           }
-          setState(219);
+          setState(221);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(222);
+      setState(224);
       queryNoWith();
       }
     }
@@ -1155,42 +1156,42 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(224);
+      setState(226);
       queryTerm();
-      setState(235);
+      setState(237);
       _la = _input.LA(1);
       if (_la==ORDER) {
         {
-        setState(225);
-        match(ORDER);
-        setState(226);
-        match(BY);
         setState(227);
+        match(ORDER);
+        setState(228);
+        match(BY);
+        setState(229);
         orderBy();
-        setState(232);
+        setState(234);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==T__2) {
           {
           {
-          setState(228);
+          setState(230);
           match(T__2);
-          setState(229);
+          setState(231);
           orderBy();
           }
           }
-          setState(234);
+          setState(236);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(238);
+      setState(240);
       _la = _input.LA(1);
       if (_la==LIMIT || _la==LIMIT_ESC) {
         {
-        setState(237);
+        setState(239);
         limitClause();
         }
       }
@@ -1239,14 +1240,14 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 10, RULE_limitClause);
     int _la;
     try {
-      setState(245);
+      setState(247);
       switch (_input.LA(1)) {
       case LIMIT:
         enterOuterAlt(_localctx, 1);
         {
-        setState(240);
+        setState(242);
         match(LIMIT);
-        setState(241);
+        setState(243);
         ((LimitClauseContext)_localctx).limit = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==ALL || _la==INTEGER_VALUE) ) {
@@ -1259,9 +1260,9 @@ class SqlBaseParser extends Parser {
       case LIMIT_ESC:
         enterOuterAlt(_localctx, 2);
         {
-        setState(242);
+        setState(244);
         match(LIMIT_ESC);
-        setState(243);
+        setState(245);
         ((LimitClauseContext)_localctx).limit = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==ALL || _la==INTEGER_VALUE) ) {
@@ -1269,7 +1270,7 @@ class SqlBaseParser extends Parser {
         } else {
           consume();
         }
-        setState(244);
+        setState(246);
         match(ESC_END);
         }
         break;
@@ -1342,13 +1343,13 @@ class SqlBaseParser extends Parser {
     QueryTermContext _localctx = new QueryTermContext(_ctx, getState());
     enterRule(_localctx, 12, RULE_queryTerm);
     try {
-      setState(252);
+      setState(254);
       switch (_input.LA(1)) {
       case SELECT:
         _localctx = new QueryPrimaryDefaultContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(247);
+        setState(249);
         querySpecification();
         }
         break;
@@ -1356,11 +1357,11 @@ class SqlBaseParser extends Parser {
         _localctx = new SubqueryContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(248);
-        match(T__0);
-        setState(249);
-        queryNoWith();
         setState(250);
+        match(T__0);
+        setState(251);
+        queryNoWith();
+        setState(252);
         match(T__1);
         }
         break;
@@ -1412,13 +1413,13 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(254);
-      expression();
       setState(256);
+      expression();
+      setState(258);
       _la = _input.LA(1);
       if (_la==ASC || _la==DESC) {
         {
-        setState(255);
+        setState(257);
         ((OrderByContext)_localctx).ordering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==ASC || _la==DESC) ) {
@@ -1497,75 +1498,75 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(258);
-      match(SELECT);
       setState(260);
+      match(SELECT);
+      setState(262);
       _la = _input.LA(1);
       if (_la==ALL || _la==DISTINCT) {
         {
-        setState(259);
+        setState(261);
         setQuantifier();
         }
       }
 
-      setState(262);
+      setState(264);
       selectItem();
-      setState(267);
+      setState(269);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==T__2) {
         {
         {
-        setState(263);
+        setState(265);
         match(T__2);
-        setState(264);
+        setState(266);
         selectItem();
         }
         }
-        setState(269);
+        setState(271);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
-      setState(271);
+      setState(273);
       _la = _input.LA(1);
       if (_la==FROM) {
         {
-        setState(270);
+        setState(272);
         fromClause();
         }
       }
 
-      setState(275);
+      setState(277);
       _la = _input.LA(1);
       if (_la==WHERE) {
         {
-        setState(273);
+        setState(275);
         match(WHERE);
-        setState(274);
+        setState(276);
         ((QuerySpecificationContext)_localctx).where = booleanExpression(0);
         }
       }
 
-      setState(280);
+      setState(282);
       _la = _input.LA(1);
       if (_la==GROUP) {
         {
-        setState(277);
-        match(GROUP);
-        setState(278);
-        match(BY);
         setState(279);
+        match(GROUP);
+        setState(280);
+        match(BY);
+        setState(281);
         groupBy();
         }
       }
 
-      setState(284);
+      setState(286);
       _la = _input.LA(1);
       if (_la==HAVING) {
         {
-        setState(282);
+        setState(284);
         match(HAVING);
-        setState(283);
+        setState(285);
         ((QuerySpecificationContext)_localctx).having = booleanExpression(0);
         }
       }
@@ -1617,23 +1618,23 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(286);
+      setState(288);
       match(FROM);
-      setState(287);
+      setState(289);
       relation();
-      setState(292);
+      setState(294);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==T__2) {
         {
         {
-        setState(288);
+        setState(290);
         match(T__2);
-        setState(289);
+        setState(291);
         relation();
         }
         }
-        setState(294);
+        setState(296);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1686,30 +1687,30 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(296);
+      setState(298);
       _la = _input.LA(1);
       if (_la==ALL || _la==DISTINCT) {
         {
-        setState(295);
+        setState(297);
         setQuantifier();
         }
       }
 
-      setState(298);
+      setState(300);
       groupingElement();
-      setState(303);
+      setState(305);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==T__2) {
         {
         {
-        setState(299);
+        setState(301);
         match(T__2);
-        setState(300);
+        setState(302);
         groupingElement();
         }
         }
-        setState(305);
+        setState(307);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1764,7 +1765,7 @@ class SqlBaseParser extends Parser {
       _localctx = new SingleGroupingSetContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(306);
+      setState(308);
       groupingExpressions();
       }
     }
@@ -1810,47 +1811,47 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 24, RULE_groupingExpressions);
     int _la;
     try {
-      setState(321);
+      setState(323);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(308);
+        setState(310);
         match(T__0);
-        setState(317);
+        setState(319);
         _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << MATCH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TRUE - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (FUNCTION_ESC - 64)) | (1L << (DATE_ESC - 64)) | (1L << (TIME_ESC - 64)) | (1L << (TIMESTAMP_ESC - 64)) | (1L << (GUID_ESC - 64)) | (1L << (PLUS - 64)) | (1L << (MINUS - 64)) | (1L << (ASTERISK - 64)) | (1L << (PARAM - 64)) | (1L << (STRING - 64)) | (1L << (INTEGER_VALUE - 64)) | (1L << (DECIMAL_VALUE - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
+        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << LEFT) | (1L << MAPPED) | (1L << MATCH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RIGHT) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TRUE - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (FUNCTION_ESC - 64)) | (1L << (DATE_ESC - 64)) | (1L << (TIME_ESC - 64)) | (1L << (TIMESTAMP_ESC - 64)) | (1L << (GUID_ESC - 64)) | (1L << (PLUS - 64)) | (1L << (MINUS - 64)) | (1L << (ASTERISK - 64)) | (1L << (PARAM - 64)) | (1L << (STRING - 64)) | (1L << (INTEGER_VALUE - 64)) | (1L << (DECIMAL_VALUE - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
           {
-          setState(309);
+          setState(311);
           expression();
-          setState(314);
+          setState(316);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==T__2) {
             {
             {
-            setState(310);
+            setState(312);
             match(T__2);
-            setState(311);
+            setState(313);
             expression();
             }
             }
-            setState(316);
+            setState(318);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(319);
+        setState(321);
         match(T__1);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(320);
+        setState(322);
         expression();
         }
         break;
@@ -1901,15 +1902,15 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(323);
-      ((NamedQueryContext)_localctx).name = identifier();
-      setState(324);
-      match(AS);
       setState(325);
-      match(T__0);
+      ((NamedQueryContext)_localctx).name = identifier();
       setState(326);
-      queryNoWith();
+      match(AS);
       setState(327);
+      match(T__0);
+      setState(328);
+      queryNoWith();
+      setState(329);
       match(T__1);
       }
     }
@@ -1953,7 +1954,7 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(329);
+      setState(331);
       _la = _input.LA(1);
       if ( !(_la==ALL || _la==DISTINCT) ) {
       _errHandler.recoverInline(this);
@@ -2016,22 +2017,22 @@ class SqlBaseParser extends Parser {
       _localctx = new SelectExpressionContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(331);
+      setState(333);
       expression();
-      setState(336);
+      setState(338);
       _la = _input.LA(1);
       if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
         {
-        setState(333);
+        setState(335);
         _la = _input.LA(1);
         if (_la==AS) {
           {
-          setState(332);
+          setState(334);
           match(AS);
           }
         }
 
-        setState(335);
+        setState(337);
         identifier();
         }
       }
@@ -2085,19 +2086,19 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(338);
+      setState(340);
       relationPrimary();
-      setState(342);
+      setState(344);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << FULL) | (1L << INNER) | (1L << JOIN) | (1L << LEFT) | (1L << NATURAL) | (1L << RIGHT))) != 0)) {
         {
         {
-        setState(339);
+        setState(341);
         joinRelation();
         }
         }
-        setState(344);
+        setState(346);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -2151,7 +2152,7 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 34, RULE_joinRelation);
     int _la;
     try {
-      setState(356);
+      setState(358);
       switch (_input.LA(1)) {
       case FULL:
       case INNER:
@@ -2161,18 +2162,18 @@ class SqlBaseParser extends Parser {
         enterOuterAlt(_localctx, 1);
         {
         {
-        setState(345);
+        setState(347);
         joinType();
         }
-        setState(346);
+        setState(348);
         match(JOIN);
-        setState(347);
-        ((JoinRelationContext)_localctx).right = relationPrimary();
         setState(349);
+        ((JoinRelationContext)_localctx).right = relationPrimary();
+        setState(351);
         _la = _input.LA(1);
         if (_la==ON || _la==USING) {
           {
-          setState(348);
+          setState(350);
           joinCriteria();
           }
         }
@@ -2182,13 +2183,13 @@ class SqlBaseParser extends Parser {
       case NATURAL:
         enterOuterAlt(_localctx, 2);
         {
-        setState(351);
-        match(NATURAL);
-        setState(352);
-        joinType();
         setState(353);
-        match(JOIN);
+        match(NATURAL);
         setState(354);
+        joinType();
+        setState(355);
+        match(JOIN);
+        setState(356);
         ((JoinRelationContext)_localctx).right = relationPrimary();
         }
         break;
@@ -2237,17 +2238,17 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 36, RULE_joinType);
     int _la;
     try {
-      setState(373);
+      setState(375);
       switch (_input.LA(1)) {
       case INNER:
       case JOIN:
         enterOuterAlt(_localctx, 1);
         {
-        setState(359);
+        setState(361);
         _la = _input.LA(1);
         if (_la==INNER) {
           {
-          setState(358);
+          setState(360);
           match(INNER);
           }
         }
@@ -2257,13 +2258,13 @@ class SqlBaseParser extends Parser {
       case LEFT:
         enterOuterAlt(_localctx, 2);
         {
-        setState(361);
-        match(LEFT);
         setState(363);
+        match(LEFT);
+        setState(365);
         _la = _input.LA(1);
         if (_la==OUTER) {
           {
-          setState(362);
+          setState(364);
           match(OUTER);
           }
         }
@@ -2273,13 +2274,13 @@ class SqlBaseParser extends Parser {
       case RIGHT:
         enterOuterAlt(_localctx, 3);
         {
-        setState(365);
-        match(RIGHT);
         setState(367);
+        match(RIGHT);
+        setState(369);
         _la = _input.LA(1);
         if (_la==OUTER) {
           {
-          setState(366);
+          setState(368);
           match(OUTER);
           }
         }
@@ -2289,13 +2290,13 @@ class SqlBaseParser extends Parser {
       case FULL:
         enterOuterAlt(_localctx, 4);
         {
-        setState(369);
-        match(FULL);
         setState(371);
+        match(FULL);
+        setState(373);
         _la = _input.LA(1);
         if (_la==OUTER) {
           {
-          setState(370);
+          setState(372);
           match(OUTER);
           }
         }
@@ -2353,43 +2354,43 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 38, RULE_joinCriteria);
     int _la;
     try {
-      setState(389);
+      setState(391);
       switch (_input.LA(1)) {
       case ON:
         enterOuterAlt(_localctx, 1);
         {
-        setState(375);
+        setState(377);
         match(ON);
-        setState(376);
+        setState(378);
         booleanExpression(0);
         }
         break;
       case USING:
         enterOuterAlt(_localctx, 2);
         {
-        setState(377);
-        match(USING);
-        setState(378);
-        match(T__0);
         setState(379);
+        match(USING);
+        setState(380);
+        match(T__0);
+        setState(381);
         identifier();
-        setState(384);
+        setState(386);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==T__2) {
           {
           {
-          setState(380);
+          setState(382);
           match(T__2);
-          setState(381);
+          setState(383);
           identifier();
           }
           }
-          setState(386);
+          setState(388);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(387);
+        setState(389);
         match(T__1);
         }
         break;
@@ -2494,29 +2495,29 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 40, RULE_relationPrimary);
     int _la;
     try {
-      setState(416);
+      setState(418);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,60,_ctx) ) {
       case 1:
         _localctx = new TableNameContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(391);
+        setState(393);
         tableIdentifier();
-        setState(396);
+        setState(398);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
           {
-          setState(393);
+          setState(395);
           _la = _input.LA(1);
           if (_la==AS) {
             {
-            setState(392);
+            setState(394);
             match(AS);
             }
           }
 
-          setState(395);
+          setState(397);
           qualifiedName();
           }
         }
@@ -2527,26 +2528,26 @@ class SqlBaseParser extends Parser {
         _localctx = new AliasedQueryContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(398);
-        match(T__0);
-        setState(399);
-        queryNoWith();
         setState(400);
+        match(T__0);
+        setState(401);
+        queryNoWith();
+        setState(402);
         match(T__1);
-        setState(405);
+        setState(407);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
           {
-          setState(402);
+          setState(404);
           _la = _input.LA(1);
           if (_la==AS) {
             {
-            setState(401);
+            setState(403);
             match(AS);
             }
           }
 
-          setState(404);
+          setState(406);
           qualifiedName();
           }
         }
@@ -2557,26 +2558,26 @@ class SqlBaseParser extends Parser {
         _localctx = new AliasedRelationContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(407);
-        match(T__0);
-        setState(408);
-        relation();
         setState(409);
+        match(T__0);
+        setState(410);
+        relation();
+        setState(411);
         match(T__1);
-        setState(414);
+        setState(416);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
           {
-          setState(411);
+          setState(413);
           _la = _input.LA(1);
           if (_la==AS) {
             {
-            setState(410);
+            setState(412);
             match(AS);
             }
           }
 
-          setState(413);
+          setState(415);
           qualifiedName();
           }
         }
@@ -2625,7 +2626,7 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(418);
+      setState(420);
       booleanExpression(0);
       }
     }
@@ -2834,7 +2835,7 @@ class SqlBaseParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(469);
+      setState(471);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,64,_ctx) ) {
       case 1:
@@ -2843,9 +2844,9 @@ class SqlBaseParser extends Parser {
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(421);
+        setState(423);
         match(NOT);
-        setState(422);
+        setState(424);
         booleanExpression(8);
         }
         break;
@@ -2854,13 +2855,13 @@ class SqlBaseParser extends Parser {
         _localctx = new ExistsContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(423);
-        match(EXISTS);
-        setState(424);
-        match(T__0);
         setState(425);
-        query();
+        match(EXISTS);
         setState(426);
+        match(T__0);
+        setState(427);
+        query();
+        setState(428);
         match(T__1);
         }
         break;
@@ -2869,29 +2870,29 @@ class SqlBaseParser extends Parser {
         _localctx = new StringQueryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(428);
-        match(QUERY);
-        setState(429);
-        match(T__0);
         setState(430);
+        match(QUERY);
+        setState(431);
+        match(T__0);
+        setState(432);
         ((StringQueryContext)_localctx).queryString = string();
-        setState(435);
+        setState(437);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==T__2) {
           {
           {
-          setState(431);
+          setState(433);
           match(T__2);
-          setState(432);
+          setState(434);
           ((StringQueryContext)_localctx).options = string();
           }
           }
-          setState(437);
+          setState(439);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(438);
+        setState(440);
         match(T__1);
         }
         break;
@@ -2900,33 +2901,33 @@ class SqlBaseParser extends Parser {
         _localctx = new MatchQueryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(440);
-        match(MATCH);
-        setState(441);
-        match(T__0);
         setState(442);
-        ((MatchQueryContext)_localctx).singleField = qualifiedName();
+        match(MATCH);
         setState(443);
-        match(T__2);
+        match(T__0);
         setState(444);
+        ((MatchQueryContext)_localctx).singleField = qualifiedName();
+        setState(445);
+        match(T__2);
+        setState(446);
         ((MatchQueryContext)_localctx).queryString = string();
-        setState(449);
+        setState(451);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==T__2) {
           {
           {
-          setState(445);
+          setState(447);
           match(T__2);
-          setState(446);
+          setState(448);
           ((MatchQueryContext)_localctx).options = string();
           }
           }
-          setState(451);
+          setState(453);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(452);
+        setState(454);
         match(T__1);
         }
         break;
@@ -2935,33 +2936,33 @@ class SqlBaseParser extends Parser {
         _localctx = new MultiMatchQueryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(454);
-        match(MATCH);
-        setState(455);
-        match(T__0);
         setState(456);
-        ((MultiMatchQueryContext)_localctx).multiFields = string();
+        match(MATCH);
         setState(457);
-        match(T__2);
+        match(T__0);
         setState(458);
+        ((MultiMatchQueryContext)_localctx).multiFields = string();
+        setState(459);
+        match(T__2);
+        setState(460);
         ((MultiMatchQueryContext)_localctx).queryString = string();
-        setState(463);
+        setState(465);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==T__2) {
           {
           {
-          setState(459);
+          setState(461);
           match(T__2);
-          setState(460);
+          setState(462);
           ((MultiMatchQueryContext)_localctx).options = string();
           }
           }
-          setState(465);
+          setState(467);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(466);
+        setState(468);
         match(T__1);
         }
         break;
@@ -2970,13 +2971,13 @@ class SqlBaseParser extends Parser {
         _localctx = new BooleanDefaultContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(468);
+        setState(470);
         predicated();
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(479);
+      setState(481);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,66,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -2984,7 +2985,7 @@ class SqlBaseParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(477);
+          setState(479);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,65,_ctx) ) {
           case 1:
@@ -2992,11 +2993,11 @@ class SqlBaseParser extends Parser {
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(471);
-            if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(472);
-            ((LogicalBinaryContext)_localctx).operator = match(AND);
             setState(473);
+            if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
+            setState(474);
+            ((LogicalBinaryContext)_localctx).operator = match(AND);
+            setState(475);
             ((LogicalBinaryContext)_localctx).right = booleanExpression(3);
             }
             break;
@@ -3005,18 +3006,18 @@ class SqlBaseParser extends Parser {
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(474);
-            if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-            setState(475);
-            ((LogicalBinaryContext)_localctx).operator = match(OR);
             setState(476);
+            if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
+            setState(477);
+            ((LogicalBinaryContext)_localctx).operator = match(OR);
+            setState(478);
             ((LogicalBinaryContext)_localctx).right = booleanExpression(2);
             }
             break;
           }
           } 
         }
-        setState(481);
+        setState(483);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,66,_ctx);
       }
@@ -3065,14 +3066,14 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(482);
-      valueExpression(0);
       setState(484);
+      valueExpression(0);
+      setState(486);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,67,_ctx) ) {
       case 1:
         {
-        setState(483);
+        setState(485);
         predicate();
         }
         break;
@@ -3148,142 +3149,142 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 48, RULE_predicate);
     int _la;
     try {
-      setState(532);
+      setState(534);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,75,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(487);
+        setState(489);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(486);
+          setState(488);
           match(NOT);
           }
         }
 
-        setState(489);
-        ((PredicateContext)_localctx).kind = match(BETWEEN);
-        setState(490);
-        ((PredicateContext)_localctx).lower = valueExpression(0);
         setState(491);
-        match(AND);
+        ((PredicateContext)_localctx).kind = match(BETWEEN);
         setState(492);
+        ((PredicateContext)_localctx).lower = valueExpression(0);
+        setState(493);
+        match(AND);
+        setState(494);
         ((PredicateContext)_localctx).upper = valueExpression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(495);
+        setState(497);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(494);
+          setState(496);
           match(NOT);
           }
         }
 
-        setState(497);
-        ((PredicateContext)_localctx).kind = match(IN);
-        setState(498);
-        match(T__0);
         setState(499);
+        ((PredicateContext)_localctx).kind = match(IN);
+        setState(500);
+        match(T__0);
+        setState(501);
         expression();
-        setState(504);
+        setState(506);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==T__2) {
           {
           {
-          setState(500);
+          setState(502);
           match(T__2);
-          setState(501);
+          setState(503);
           expression();
           }
           }
-          setState(506);
+          setState(508);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(507);
+        setState(509);
         match(T__1);
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(510);
+        setState(512);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(509);
+          setState(511);
           match(NOT);
           }
         }
 
-        setState(512);
-        ((PredicateContext)_localctx).kind = match(IN);
-        setState(513);
-        match(T__0);
         setState(514);
-        query();
+        ((PredicateContext)_localctx).kind = match(IN);
         setState(515);
+        match(T__0);
+        setState(516);
+        query();
+        setState(517);
         match(T__1);
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(518);
+        setState(520);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(517);
+          setState(519);
           match(NOT);
           }
         }
 
-        setState(520);
+        setState(522);
         ((PredicateContext)_localctx).kind = match(LIKE);
-        setState(521);
+        setState(523);
         pattern();
         }
         break;
       case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(523);
+        setState(525);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(522);
+          setState(524);
           match(NOT);
           }
         }
 
-        setState(525);
+        setState(527);
         ((PredicateContext)_localctx).kind = match(RLIKE);
-        setState(526);
+        setState(528);
         ((PredicateContext)_localctx).regex = string();
         }
         break;
       case 6:
         enterOuterAlt(_localctx, 6);
         {
-        setState(527);
-        match(IS);
         setState(529);
+        match(IS);
+        setState(531);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(528);
+          setState(530);
           match(NOT);
           }
         }
 
-        setState(531);
+        setState(533);
         ((PredicateContext)_localctx).kind = match(NULL);
         }
         break;
@@ -3333,14 +3334,14 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(534);
-      ((PatternContext)_localctx).value = string();
       setState(536);
+      ((PatternContext)_localctx).value = string();
+      setState(538);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,76,_ctx) ) {
       case 1:
         {
-        setState(535);
+        setState(537);
         patternEscape();
         }
         break;
@@ -3388,25 +3389,25 @@ class SqlBaseParser extends Parser {
     PatternEscapeContext _localctx = new PatternEscapeContext(_ctx, getState());
     enterRule(_localctx, 52, RULE_patternEscape);
     try {
-      setState(544);
+      setState(546);
       switch (_input.LA(1)) {
       case ESCAPE:
         enterOuterAlt(_localctx, 1);
         {
-        setState(538);
+        setState(540);
         match(ESCAPE);
-        setState(539);
+        setState(541);
         ((PatternEscapeContext)_localctx).escape = string();
         }
         break;
       case ESCAPE_ESC:
         enterOuterAlt(_localctx, 2);
         {
-        setState(540);
-        match(ESCAPE_ESC);
-        setState(541);
-        ((PatternEscapeContext)_localctx).escape = string();
         setState(542);
+        match(ESCAPE_ESC);
+        setState(543);
+        ((PatternEscapeContext)_localctx).escape = string();
+        setState(544);
         match(ESC_END);
         }
         break;
@@ -3551,7 +3552,7 @@ class SqlBaseParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(550);
+      setState(552);
       switch (_input.LA(1)) {
       case T__0:
       case ANALYZE:
@@ -3567,12 +3568,14 @@ class SqlBaseParser extends Parser {
       case FORMAT:
       case FUNCTIONS:
       case GRAPHVIZ:
+      case LEFT:
       case MAPPED:
       case NULL:
       case OPTIMIZED:
       case PARSED:
       case PHYSICAL:
       case PLAN:
+      case RIGHT:
       case RLIKE:
       case QUERY:
       case SCHEMAS:
@@ -3603,7 +3606,7 @@ class SqlBaseParser extends Parser {
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(547);
+        setState(549);
         primaryExpression();
         }
         break;
@@ -3613,7 +3616,7 @@ class SqlBaseParser extends Parser {
         _localctx = new ArithmeticUnaryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(548);
+        setState(550);
         ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
@@ -3621,7 +3624,7 @@ class SqlBaseParser extends Parser {
         } else {
           consume();
         }
-        setState(549);
+        setState(551);
         valueExpression(4);
         }
         break;
@@ -3629,7 +3632,7 @@ class SqlBaseParser extends Parser {
         throw new NoViableAltException(this);
       }
       _ctx.stop = _input.LT(-1);
-      setState(564);
+      setState(566);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,80,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -3637,7 +3640,7 @@ class SqlBaseParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(562);
+          setState(564);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,79,_ctx) ) {
           case 1:
@@ -3645,9 +3648,9 @@ class SqlBaseParser extends Parser {
             _localctx = new ArithmeticBinaryContext(new ValueExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_valueExpression);
-            setState(552);
+            setState(554);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(553);
+            setState(555);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
             if ( !(((((_la - 88)) & ~0x3f) == 0 && ((1L << (_la - 88)) & ((1L << (ASTERISK - 88)) | (1L << (SLASH - 88)) | (1L << (PERCENT - 88)))) != 0)) ) {
@@ -3655,7 +3658,7 @@ class SqlBaseParser extends Parser {
             } else {
               consume();
             }
-            setState(554);
+            setState(556);
             ((ArithmeticBinaryContext)_localctx).right = valueExpression(4);
             }
             break;
@@ -3664,9 +3667,9 @@ class SqlBaseParser extends Parser {
             _localctx = new ArithmeticBinaryContext(new ValueExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_valueExpression);
-            setState(555);
+            setState(557);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(556);
+            setState(558);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
             if ( !(_la==PLUS || _la==MINUS) ) {
@@ -3674,7 +3677,7 @@ class SqlBaseParser extends Parser {
             } else {
               consume();
             }
-            setState(557);
+            setState(559);
             ((ArithmeticBinaryContext)_localctx).right = valueExpression(3);
             }
             break;
@@ -3683,18 +3686,18 @@ class SqlBaseParser extends Parser {
             _localctx = new ComparisonContext(new ValueExpressionContext(_parentctx, _parentState));
             ((ComparisonContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_valueExpression);
-            setState(558);
-            if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-            setState(559);
-            comparisonOperator();
             setState(560);
+            if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
+            setState(561);
+            comparisonOperator();
+            setState(562);
             ((ComparisonContext)_localctx).right = valueExpression(2);
             }
             break;
           }
           } 
         }
-        setState(566);
+        setState(568);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,80,_ctx);
       }
@@ -3901,14 +3904,14 @@ class SqlBaseParser extends Parser {
     enterRule(_localctx, 56, RULE_primaryExpression);
     int _la;
     try {
-      setState(588);
+      setState(590);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,82,_ctx) ) {
       case 1:
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(567);
+        setState(569);
         castExpression();
         }
         break;
@@ -3916,7 +3919,7 @@ class SqlBaseParser extends Parser {
         _localctx = new ExtractContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(568);
+        setState(570);
         extractExpression();
         }
         break;
@@ -3924,7 +3927,7 @@ class SqlBaseParser extends Parser {
         _localctx = new ConstantDefaultContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(569);
+        setState(571);
         constant();
         }
         break;
@@ -3932,7 +3935,7 @@ class SqlBaseParser extends Parser {
         _localctx = new StarContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(570);
+        setState(572);
         match(ASTERISK);
         }
         break;
@@ -3940,18 +3943,18 @@ class SqlBaseParser extends Parser {
         _localctx = new StarContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(574);
+        setState(576);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
           {
-          setState(571);
+          setState(573);
           qualifiedName();
-          setState(572);
+          setState(574);
           match(DOT);
           }
         }
 
-        setState(576);
+        setState(578);
         match(ASTERISK);
         }
         break;
@@ -3959,7 +3962,7 @@ class SqlBaseParser extends Parser {
         _localctx = new FunctionContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(577);
+        setState(579);
         functionExpression();
         }
         break;
@@ -3967,11 +3970,11 @@ class SqlBaseParser extends Parser {
         _localctx = new SubqueryExpressionContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(578);
-        match(T__0);
-        setState(579);
-        query();
         setState(580);
+        match(T__0);
+        setState(581);
+        query();
+        setState(582);
         match(T__1);
         }
         break;
@@ -3979,7 +3982,7 @@ class SqlBaseParser extends Parser {
         _localctx = new ColumnReferenceContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(582);
+        setState(584);
         identifier();
         }
         break;
@@ -3987,7 +3990,7 @@ class SqlBaseParser extends Parser {
         _localctx = new DereferenceContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(583);
+        setState(585);
         qualifiedName();
         }
         break;
@@ -3995,11 +3998,11 @@ class SqlBaseParser extends Parser {
         _localctx = new ParenthesizedExpressionContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(584);
-        match(T__0);
-        setState(585);
-        expression();
         setState(586);
+        match(T__0);
+        setState(587);
+        expression();
+        setState(588);
         match(T__1);
         }
         break;
@@ -4045,23 +4048,23 @@ class SqlBaseParser extends Parser {
     CastExpressionContext _localctx = new CastExpressionContext(_ctx, getState());
     enterRule(_localctx, 58, RULE_castExpression);
     try {
-      setState(595);
+      setState(597);
       switch (_input.LA(1)) {
       case CAST:
         enterOuterAlt(_localctx, 1);
         {
-        setState(590);
+        setState(592);
         castTemplate();
         }
         break;
       case FUNCTION_ESC:
         enterOuterAlt(_localctx, 2);
         {
-        setState(591);
-        match(FUNCTION_ESC);
-        setState(592);
-        castTemplate();
         setState(593);
+        match(FUNCTION_ESC);
+        setState(594);
+        castTemplate();
+        setState(595);
         match(ESC_END);
         }
         break;
@@ -4114,17 +4117,17 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(597);
-      match(CAST);
-      setState(598);
-      match(T__0);
       setState(599);
-      expression();
+      match(CAST);
       setState(600);
-      match(AS);
+      match(T__0);
       setState(601);
-      dataType();
+      expression();
       setState(602);
+      match(AS);
+      setState(603);
+      dataType();
+      setState(604);
       match(T__1);
       }
     }
@@ -4168,23 +4171,23 @@ class SqlBaseParser extends Parser {
     ExtractExpressionContext _localctx = new ExtractExpressionContext(_ctx, getState());
     enterRule(_localctx, 62, RULE_extractExpression);
     try {
-      setState(609);
+      setState(611);
       switch (_input.LA(1)) {
       case EXTRACT:
         enterOuterAlt(_localctx, 1);
         {
-        setState(604);
+        setState(606);
         extractTemplate();
         }
         break;
       case FUNCTION_ESC:
         enterOuterAlt(_localctx, 2);
         {
-        setState(605);
-        match(FUNCTION_ESC);
-        setState(606);
-        extractTemplate();
         setState(607);
+        match(FUNCTION_ESC);
+        setState(608);
+        extractTemplate();
+        setState(609);
         match(ESC_END);
         }
         break;
@@ -4238,17 +4241,17 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(611);
-      match(EXTRACT);
-      setState(612);
-      match(T__0);
       setState(613);
-      ((ExtractTemplateContext)_localctx).field = identifier();
+      match(EXTRACT);
       setState(614);
-      match(FROM);
+      match(T__0);
       setState(615);
-      valueExpression(0);
+      ((ExtractTemplateContext)_localctx).field = identifier();
       setState(616);
+      match(FROM);
+      setState(617);
+      valueExpression(0);
+      setState(618);
       match(T__1);
       }
     }
@@ -4291,7 +4294,7 @@ class SqlBaseParser extends Parser {
     FunctionExpressionContext _localctx = new FunctionExpressionContext(_ctx, getState());
     enterRule(_localctx, 66, RULE_functionExpression);
     try {
-      setState(623);
+      setState(625);
       switch (_input.LA(1)) {
       case ANALYZE:
       case ANALYZED:
@@ -4303,11 +4306,13 @@ class SqlBaseParser extends Parser {
       case FORMAT:
       case FUNCTIONS:
       case GRAPHVIZ:
+      case LEFT:
       case MAPPED:
       case OPTIMIZED:
       case PARSED:
       case PHYSICAL:
       case PLAN:
+      case RIGHT:
       case RLIKE:
       case QUERY:
       case SCHEMAS:
@@ -4324,18 +4329,18 @@ class SqlBaseParser extends Parser {
       case BACKQUOTED_IDENTIFIER:
         enterOuterAlt(_localctx, 1);
         {
-        setState(618);
+        setState(620);
         functionTemplate();
         }
         break;
       case FUNCTION_ESC:
         enterOuterAlt(_localctx, 2);
         {
-        setState(619);
-        match(FUNCTION_ESC);
-        setState(620);
-        functionTemplate();
         setState(621);
+        match(FUNCTION_ESC);
+        setState(622);
+        functionTemplate();
+        setState(623);
         match(ESC_END);
         }
         break;
@@ -4355,8 +4360,8 @@ class SqlBaseParser extends Parser {
   }
 
   public static class FunctionTemplateContext extends ParserRuleContext {
-    public IdentifierContext identifier() {
-      return getRuleContext(IdentifierContext.class,0);
+    public FunctionNameContext functionName() {
+      return getRuleContext(FunctionNameContext.class,0);
     }
     public List<ExpressionContext> expression() {
       return getRuleContexts(ExpressionContext.class);
@@ -4393,45 +4398,45 @@ class SqlBaseParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(625);
-      identifier();
-      setState(626);
+      setState(627);
+      functionName();
+      setState(628);
       match(T__0);
-      setState(638);
+      setState(640);
       _la = _input.LA(1);
-      if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ALL) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << DISTINCT) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << MATCH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TRUE - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (FUNCTION_ESC - 64)) | (1L << (DATE_ESC - 64)) | (1L << (TIME_ESC - 64)) | (1L << (TIMESTAMP_ESC - 64)) | (1L << (GUID_ESC - 64)) | (1L << (PLUS - 64)) | (1L << (MINUS - 64)) | (1L << (ASTERISK - 64)) | (1L << (PARAM - 64)) | (1L << (STRING - 64)) | (1L << (INTEGER_VALUE - 64)) | (1L << (DECIMAL_VALUE - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
+      if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ALL) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << DISTINCT) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << LEFT) | (1L << MAPPED) | (1L << MATCH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RIGHT) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TRUE - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (FUNCTION_ESC - 64)) | (1L << (DATE_ESC - 64)) | (1L << (TIME_ESC - 64)) | (1L << (TIMESTAMP_ESC - 64)) | (1L << (GUID_ESC - 64)) | (1L << (PLUS - 64)) | (1L << (MINUS - 64)) | (1L << (ASTERISK - 64)) | (1L << (PARAM - 64)) | (1L << (STRING - 64)) | (1L << (INTEGER_VALUE - 64)) | (1L << (DECIMAL_VALUE - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
         {
-        setState(628);
+        setState(630);
         _la = _input.LA(1);
         if (_la==ALL || _la==DISTINCT) {
           {
-          setState(627);
+          setState(629);
           setQuantifier();
           }
         }
 
-        setState(630);
+        setState(632);
         expression();
-        setState(635);
+        setState(637);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==T__2) {
           {
           {
-          setState(631);
+          setState(633);
           match(T__2);
-          setState(632);
+          setState(634);
           expression();
           }
           }
-          setState(637);
+          setState(639);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(640);
+      setState(642);
       match(T__1);
       }
     }
@@ -4446,6 +4451,101 @@ class SqlBaseParser extends Parser {
     return _localctx;
   }
 
+  public static class FunctionNameContext extends ParserRuleContext {
+    public TerminalNode LEFT() { return getToken(SqlBaseParser.LEFT, 0); }
+    public TerminalNode RIGHT() { return getToken(SqlBaseParser.RIGHT, 0); }
+    public IdentifierContext identifier() {
+      return getRuleContext(IdentifierContext.class,0);
+    }
+    public FunctionNameContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_functionName; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterFunctionName(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitFunctionName(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor<? extends T>)visitor).visitFunctionName(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final FunctionNameContext functionName() throws RecognitionException {
+    FunctionNameContext _localctx = new FunctionNameContext(_ctx, getState());
+    enterRule(_localctx, 70, RULE_functionName);
+    try {
+      setState(647);
+      switch (_input.LA(1)) {
+      case LEFT:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(644);
+        match(LEFT);
+        }
+        break;
+      case RIGHT:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(645);
+        match(RIGHT);
+        }
+        break;
+      case ANALYZE:
+      case ANALYZED:
+      case CATALOGS:
+      case COLUMNS:
+      case DEBUG:
+      case EXECUTABLE:
+      case EXPLAIN:
+      case FORMAT:
+      case FUNCTIONS:
+      case GRAPHVIZ:
+      case MAPPED:
+      case OPTIMIZED:
+      case PARSED:
+      case PHYSICAL:
+      case PLAN:
+      case RLIKE:
+      case QUERY:
+      case SCHEMAS:
+      case SHOW:
+      case SYS:
+      case TABLES:
+      case TEXT:
+      case TYPE:
+      case TYPES:
+      case VERIFY:
+      case IDENTIFIER:
+      case DIGIT_IDENTIFIER:
+      case QUOTED_IDENTIFIER:
+      case BACKQUOTED_IDENTIFIER:
+        enterOuterAlt(_localctx, 3);
+        {
+        setState(646);
+        identifier();
+        }
+        break;
+      default:
+        throw new NoViableAltException(this);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   public static class ConstantContext extends ParserRuleContext {
     public ConstantContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
@@ -4636,16 +4736,16 @@ class SqlBaseParser extends Parser {
 
   public final ConstantContext constant() throws RecognitionException {
     ConstantContext _localctx = new ConstantContext(_ctx, getState());
-    enterRule(_localctx, 70, RULE_constant);
+    enterRule(_localctx, 72, RULE_constant);
     try {
       int _alt;
-      setState(667);
+      setState(674);
       switch (_input.LA(1)) {
       case NULL:
         _localctx = new NullLiteralContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(642);
+        setState(649);
         match(NULL);
         }
         break;
@@ -4654,7 +4754,7 @@ class SqlBaseParser extends Parser {
         _localctx = new NumericLiteralContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(643);
+        setState(650);
         number();
         }
         break;
@@ -4663,7 +4763,7 @@ class SqlBaseParser extends Parser {
         _localctx = new BooleanLiteralContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(644);
+        setState(651);
         booleanValue();
         }
         break;
@@ -4671,7 +4771,7 @@ class SqlBaseParser extends Parser {
         _localctx = new StringLiteralContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(646); 
+        setState(653); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -4679,7 +4779,7 @@ class SqlBaseParser extends Parser {
           case 1:
             {
             {
-            setState(645);
+            setState(652);
             match(STRING);
             }
             }
@@ -4687,9 +4787,9 @@ class SqlBaseParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(648); 
+          setState(655); 
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,89,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,90,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
         }
         break;
@@ -4697,7 +4797,7 @@ class SqlBaseParser extends Parser {
         _localctx = new ParamLiteralContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(650);
+        setState(657);
         match(PARAM);
         }
         break;
@@ -4705,11 +4805,11 @@ class SqlBaseParser extends Parser {
         _localctx = new DateEscapedLiteralContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(651);
+        setState(658);
         match(DATE_ESC);
-        setState(652);
+        setState(659);
         string();
-        setState(653);
+        setState(660);
         match(ESC_END);
         }
         break;
@@ -4717,11 +4817,11 @@ class SqlBaseParser extends Parser {
         _localctx = new TimeEscapedLiteralContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(655);
+        setState(662);
         match(TIME_ESC);
-        setState(656);
+        setState(663);
         string();
-        setState(657);
+        setState(664);
         match(ESC_END);
         }
         break;
@@ -4729,11 +4829,11 @@ class SqlBaseParser extends Parser {
         _localctx = new TimestampEscapedLiteralContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(659);
+        setState(666);
         match(TIMESTAMP_ESC);
-        setState(660);
+        setState(667);
         string();
-        setState(661);
+        setState(668);
         match(ESC_END);
         }
         break;
@@ -4741,11 +4841,11 @@ class SqlBaseParser extends Parser {
         _localctx = new GuidEscapedLiteralContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(663);
+        setState(670);
         match(GUID_ESC);
-        setState(664);
+        setState(671);
         string();
-        setState(665);
+        setState(672);
         match(ESC_END);
         }
         break;
@@ -4792,12 +4892,12 @@ class SqlBaseParser extends Parser {
 
   public final ComparisonOperatorContext comparisonOperator() throws RecognitionException {
     ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState());
-    enterRule(_localctx, 72, RULE_comparisonOperator);
+    enterRule(_localctx, 74, RULE_comparisonOperator);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(669);
+      setState(676);
       _la = _input.LA(1);
       if ( !(((((_la - 80)) & ~0x3f) == 0 && ((1L << (_la - 80)) & ((1L << (EQ - 80)) | (1L << (NEQ - 80)) | (1L << (LT - 80)) | (1L << (LTE - 80)) | (1L << (GT - 80)) | (1L << (GTE - 80)))) != 0)) ) {
       _errHandler.recoverInline(this);
@@ -4841,12 +4941,12 @@ class SqlBaseParser extends Parser {
 
   public final BooleanValueContext booleanValue() throws RecognitionException {
     BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState());
-    enterRule(_localctx, 74, RULE_booleanValue);
+    enterRule(_localctx, 76, RULE_booleanValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(671);
+      setState(678);
       _la = _input.LA(1);
       if ( !(_la==FALSE || _la==TRUE) ) {
       _errHandler.recoverInline(this);
@@ -4899,12 +4999,12 @@ class SqlBaseParser extends Parser {
 
   public final DataTypeContext dataType() throws RecognitionException {
     DataTypeContext _localctx = new DataTypeContext(_ctx, getState());
-    enterRule(_localctx, 76, RULE_dataType);
+    enterRule(_localctx, 78, RULE_dataType);
     try {
       _localctx = new PrimitiveDataTypeContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(673);
+      setState(680);
       identifier();
       }
     }
@@ -4951,30 +5051,30 @@ class SqlBaseParser extends Parser {
 
   public final QualifiedNameContext qualifiedName() throws RecognitionException {
     QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState());
-    enterRule(_localctx, 78, RULE_qualifiedName);
+    enterRule(_localctx, 80, RULE_qualifiedName);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(680);
+      setState(687);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,91,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,92,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(675);
+          setState(682);
           identifier();
-          setState(676);
+          setState(683);
           match(DOT);
           }
           } 
         }
-        setState(682);
+        setState(689);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,91,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,92,_ctx);
       }
-      setState(683);
+      setState(690);
       identifier();
       }
     }
@@ -5017,15 +5117,15 @@ class SqlBaseParser extends Parser {
 
   public final IdentifierContext identifier() throws RecognitionException {
     IdentifierContext _localctx = new IdentifierContext(_ctx, getState());
-    enterRule(_localctx, 80, RULE_identifier);
+    enterRule(_localctx, 82, RULE_identifier);
     try {
-      setState(687);
+      setState(694);
       switch (_input.LA(1)) {
       case QUOTED_IDENTIFIER:
       case BACKQUOTED_IDENTIFIER:
         enterOuterAlt(_localctx, 1);
         {
-        setState(685);
+        setState(692);
         quoteIdentifier();
         }
         break;
@@ -5058,7 +5158,7 @@ class SqlBaseParser extends Parser {
       case DIGIT_IDENTIFIER:
         enterOuterAlt(_localctx, 2);
         {
-        setState(686);
+        setState(693);
         unquoteIdentifier();
         }
         break;
@@ -5108,46 +5208,46 @@ class SqlBaseParser extends Parser {
 
   public final TableIdentifierContext tableIdentifier() throws RecognitionException {
     TableIdentifierContext _localctx = new TableIdentifierContext(_ctx, getState());
-    enterRule(_localctx, 82, RULE_tableIdentifier);
+    enterRule(_localctx, 84, RULE_tableIdentifier);
     int _la;
     try {
-      setState(701);
+      setState(708);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,95,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,96,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(692);
+        setState(699);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) {
           {
-          setState(689);
+          setState(696);
           ((TableIdentifierContext)_localctx).catalog = identifier();
-          setState(690);
+          setState(697);
           match(T__3);
           }
         }
 
-        setState(694);
+        setState(701);
         match(TABLE_IDENTIFIER);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(698);
+        setState(705);
         _errHandler.sync(this);
-        switch ( getInterpreter().adaptivePredict(_input,94,_ctx) ) {
+        switch ( getInterpreter().adaptivePredict(_input,95,_ctx) ) {
         case 1:
           {
-          setState(695);
+          setState(702);
           ((TableIdentifierContext)_localctx).catalog = identifier();
-          setState(696);
+          setState(703);
           match(T__3);
           }
           break;
         }
-        setState(700);
+        setState(707);
         ((TableIdentifierContext)_localctx).name = identifier();
         }
         break;
@@ -5212,15 +5312,15 @@ class SqlBaseParser extends Parser {
 
   public final QuoteIdentifierContext quoteIdentifier() throws RecognitionException {
     QuoteIdentifierContext _localctx = new QuoteIdentifierContext(_ctx, getState());
-    enterRule(_localctx, 84, RULE_quoteIdentifier);
+    enterRule(_localctx, 86, RULE_quoteIdentifier);
     try {
-      setState(705);
+      setState(712);
       switch (_input.LA(1)) {
       case QUOTED_IDENTIFIER:
         _localctx = new QuotedIdentifierContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(703);
+        setState(710);
         match(QUOTED_IDENTIFIER);
         }
         break;
@@ -5228,7 +5328,7 @@ class SqlBaseParser extends Parser {
         _localctx = new BackQuotedIdentifierContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(704);
+        setState(711);
         match(BACKQUOTED_IDENTIFIER);
         }
         break;
@@ -5298,15 +5398,15 @@ class SqlBaseParser extends Parser {
 
   public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionException {
     UnquoteIdentifierContext _localctx = new UnquoteIdentifierContext(_ctx, getState());
-    enterRule(_localctx, 86, RULE_unquoteIdentifier);
+    enterRule(_localctx, 88, RULE_unquoteIdentifier);
     try {
-      setState(710);
+      setState(717);
       switch (_input.LA(1)) {
       case IDENTIFIER:
         _localctx = new UnquotedIdentifierContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(707);
+        setState(714);
         match(IDENTIFIER);
         }
         break;
@@ -5338,7 +5438,7 @@ class SqlBaseParser extends Parser {
         _localctx = new UnquotedIdentifierContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(708);
+        setState(715);
         nonReserved();
         }
         break;
@@ -5346,7 +5446,7 @@ class SqlBaseParser extends Parser {
         _localctx = new DigitIdentifierContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(709);
+        setState(716);
         match(DIGIT_IDENTIFIER);
         }
         break;
@@ -5413,15 +5513,15 @@ class SqlBaseParser extends Parser {
 
   public final NumberContext number() throws RecognitionException {
     NumberContext _localctx = new NumberContext(_ctx, getState());
-    enterRule(_localctx, 88, RULE_number);
+    enterRule(_localctx, 90, RULE_number);
     try {
-      setState(714);
+      setState(721);
       switch (_input.LA(1)) {
       case DECIMAL_VALUE:
         _localctx = new DecimalLiteralContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(712);
+        setState(719);
         match(DECIMAL_VALUE);
         }
         break;
@@ -5429,7 +5529,7 @@ class SqlBaseParser extends Parser {
         _localctx = new IntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(713);
+        setState(720);
         match(INTEGER_VALUE);
         }
         break;
@@ -5472,12 +5572,12 @@ class SqlBaseParser extends Parser {
 
   public final StringContext string() throws RecognitionException {
     StringContext _localctx = new StringContext(_ctx, getState());
-    enterRule(_localctx, 90, RULE_string);
+    enterRule(_localctx, 92, RULE_string);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(716);
+      setState(723);
       _la = _input.LA(1);
       if ( !(_la==PARAM || _la==STRING) ) {
       _errHandler.recoverInline(this);
@@ -5544,12 +5644,12 @@ class SqlBaseParser extends Parser {
 
   public final NonReservedContext nonReserved() throws RecognitionException {
     NonReservedContext _localctx = new NonReservedContext(_ctx, getState());
-    enterRule(_localctx, 92, RULE_nonReserved);
+    enterRule(_localctx, 94, RULE_nonReserved);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(718);
+      setState(725);
       _la = _input.LA(1);
       if ( !(((((_la - 6)) & ~0x3f) == 0 && ((1L << (_la - 6)) & ((1L << (ANALYZE - 6)) | (1L << (ANALYZED - 6)) | (1L << (CATALOGS - 6)) | (1L << (COLUMNS - 6)) | (1L << (DEBUG - 6)) | (1L << (EXECUTABLE - 6)) | (1L << (EXPLAIN - 6)) | (1L << (FORMAT - 6)) | (1L << (FUNCTIONS - 6)) | (1L << (GRAPHVIZ - 6)) | (1L << (MAPPED - 6)) | (1L << (OPTIMIZED - 6)) | (1L << (PARSED - 6)) | (1L << (PHYSICAL - 6)) | (1L << (PLAN - 6)) | (1L << (RLIKE - 6)) | (1L << (QUERY - 6)) | (1L << (SCHEMAS - 6)) | (1L << (SHOW - 6)) | (1L << (SYS - 6)) | (1L << (TABLES - 6)) | (1L << (TEXT - 6)) | (1L << (TYPE - 6)) | (1L << (TYPES - 6)) | (1L << (VERIFY - 6)))) != 0)) ) {
       _errHandler.recoverInline(this);
@@ -5600,291 +5700,295 @@ class SqlBaseParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3l\u02d3\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3l\u02da\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
     "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!"+
     "\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t+\4"+
-    ",\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\3\2\3\2\3\2\3\3\3\3\3\3\3\4\3\4\3\4\3"+
-    "\4\3\4\3\4\3\4\3\4\3\4\7\4p\n\4\f\4\16\4s\13\4\3\4\5\4v\n\4\3\4\3\4\3"+
-    "\4\3\4\3\4\3\4\3\4\7\4\177\n\4\f\4\16\4\u0082\13\4\3\4\5\4\u0085\n\4\3"+
-    "\4\3\4\3\4\3\4\5\4\u008b\n\4\3\4\5\4\u008e\n\4\3\4\3\4\3\4\3\4\3\4\3\4"+
-    "\3\4\3\4\3\4\5\4\u0099\n\4\3\4\5\4\u009c\n\4\3\4\3\4\3\4\3\4\3\4\3\4\3"+
-    "\4\3\4\5\4\u00a6\n\4\3\4\5\4\u00a9\n\4\3\4\5\4\u00ac\n\4\3\4\5\4\u00af"+
-    "\n\4\3\4\3\4\3\4\3\4\7\4\u00b5\n\4\f\4\16\4\u00b8\13\4\5\4\u00ba\n\4\3"+
-    "\4\3\4\3\4\3\4\5\4\u00c0\n\4\3\4\3\4\5\4\u00c4\n\4\3\4\5\4\u00c7\n\4\3"+
-    "\4\5\4\u00ca\n\4\3\4\5\4\u00cd\n\4\3\4\3\4\3\4\3\4\3\4\5\4\u00d4\n\4\3"+
-    "\5\3\5\3\5\3\5\7\5\u00da\n\5\f\5\16\5\u00dd\13\5\5\5\u00df\n\5\3\5\3\5"+
-    "\3\6\3\6\3\6\3\6\3\6\3\6\7\6\u00e9\n\6\f\6\16\6\u00ec\13\6\5\6\u00ee\n"+
-    "\6\3\6\5\6\u00f1\n\6\3\7\3\7\3\7\3\7\3\7\5\7\u00f8\n\7\3\b\3\b\3\b\3\b"+
-    "\3\b\5\b\u00ff\n\b\3\t\3\t\5\t\u0103\n\t\3\n\3\n\5\n\u0107\n\n\3\n\3\n"+
-    "\3\n\7\n\u010c\n\n\f\n\16\n\u010f\13\n\3\n\5\n\u0112\n\n\3\n\3\n\5\n\u0116"+
-    "\n\n\3\n\3\n\3\n\5\n\u011b\n\n\3\n\3\n\5\n\u011f\n\n\3\13\3\13\3\13\3"+
-    "\13\7\13\u0125\n\13\f\13\16\13\u0128\13\13\3\f\5\f\u012b\n\f\3\f\3\f\3"+
-    "\f\7\f\u0130\n\f\f\f\16\f\u0133\13\f\3\r\3\r\3\16\3\16\3\16\3\16\7\16"+
-    "\u013b\n\16\f\16\16\16\u013e\13\16\5\16\u0140\n\16\3\16\3\16\5\16\u0144"+
-    "\n\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\21\3\21\5\21\u0150\n\21"+
-    "\3\21\5\21\u0153\n\21\3\22\3\22\7\22\u0157\n\22\f\22\16\22\u015a\13\22"+
-    "\3\23\3\23\3\23\3\23\5\23\u0160\n\23\3\23\3\23\3\23\3\23\3\23\5\23\u0167"+
-    "\n\23\3\24\5\24\u016a\n\24\3\24\3\24\5\24\u016e\n\24\3\24\3\24\5\24\u0172"+
-    "\n\24\3\24\3\24\5\24\u0176\n\24\5\24\u0178\n\24\3\25\3\25\3\25\3\25\3"+
-    "\25\3\25\3\25\7\25\u0181\n\25\f\25\16\25\u0184\13\25\3\25\3\25\5\25\u0188"+
-    "\n\25\3\26\3\26\5\26\u018c\n\26\3\26\5\26\u018f\n\26\3\26\3\26\3\26\3"+
-    "\26\5\26\u0195\n\26\3\26\5\26\u0198\n\26\3\26\3\26\3\26\3\26\5\26\u019e"+
-    "\n\26\3\26\5\26\u01a1\n\26\5\26\u01a3\n\26\3\27\3\27\3\30\3\30\3\30\3"+
-    "\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\7\30\u01b4\n\30\f\30"+
-    "\16\30\u01b7\13\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\7\30\u01c2"+
-    "\n\30\f\30\16\30\u01c5\13\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3"+
-    "\30\7\30\u01d0\n\30\f\30\16\30\u01d3\13\30\3\30\3\30\3\30\5\30\u01d8\n"+
-    "\30\3\30\3\30\3\30\3\30\3\30\3\30\7\30\u01e0\n\30\f\30\16\30\u01e3\13"+
-    "\30\3\31\3\31\5\31\u01e7\n\31\3\32\5\32\u01ea\n\32\3\32\3\32\3\32\3\32"+
-    "\3\32\3\32\5\32\u01f2\n\32\3\32\3\32\3\32\3\32\3\32\7\32\u01f9\n\32\f"+
-    "\32\16\32\u01fc\13\32\3\32\3\32\3\32\5\32\u0201\n\32\3\32\3\32\3\32\3"+
-    "\32\3\32\3\32\5\32\u0209\n\32\3\32\3\32\3\32\5\32\u020e\n\32\3\32\3\32"+
-    "\3\32\3\32\5\32\u0214\n\32\3\32\5\32\u0217\n\32\3\33\3\33\5\33\u021b\n"+
-    "\33\3\34\3\34\3\34\3\34\3\34\3\34\5\34\u0223\n\34\3\35\3\35\3\35\3\35"+
-    "\5\35\u0229\n\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\7\35"+
-    "\u0235\n\35\f\35\16\35\u0238\13\35\3\36\3\36\3\36\3\36\3\36\3\36\3\36"+
-    "\5\36\u0241\n\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36"+
-    "\3\36\5\36\u024f\n\36\3\37\3\37\3\37\3\37\3\37\5\37\u0256\n\37\3 \3 \3"+
-    " \3 \3 \3 \3 \3!\3!\3!\3!\3!\5!\u0264\n!\3\"\3\"\3\"\3\"\3\"\3\"\3\"\3"+
-    "#\3#\3#\3#\3#\5#\u0272\n#\3$\3$\3$\5$\u0277\n$\3$\3$\3$\7$\u027c\n$\f"+
-    "$\16$\u027f\13$\5$\u0281\n$\3$\3$\3%\3%\3%\3%\6%\u0289\n%\r%\16%\u028a"+
-    "\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\5%\u029e\n%\3&\3&"+
-    "\3\'\3\'\3(\3(\3)\3)\3)\7)\u02a9\n)\f)\16)\u02ac\13)\3)\3)\3*\3*\5*\u02b2"+
-    "\n*\3+\3+\3+\5+\u02b7\n+\3+\3+\3+\3+\5+\u02bd\n+\3+\5+\u02c0\n+\3,\3,"+
-    "\5,\u02c4\n,\3-\3-\3-\5-\u02c9\n-\3.\3.\5.\u02cd\n.\3/\3/\3\60\3\60\3"+
-    "\60\2\4.8\61\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64"+
-    "\668:<>@BDFHJLNPRTVXZ\\^\2\20\b\2\7\7\t\t\31\31,,\62\62\66\66\4\2\"\""+
-    "BB\4\2\t\t\62\62\4\2\37\37%%\3\2\25\26\4\2\7\7aa\4\2\r\r\25\25\4\2\7\7"+
-    "\27\27\3\2XY\3\2Z\\\3\2RW\4\2\35\35CC\3\2_`\20\2\b\t\22\24\31\31\33\33"+
-    "\36\36!\",,\62\62\668:<>?ABDEGG\u032e\2`\3\2\2\2\4c\3\2\2\2\6\u00d3\3"+
-    "\2\2\2\b\u00de\3\2\2\2\n\u00e2\3\2\2\2\f\u00f7\3\2\2\2\16\u00fe\3\2\2"+
-    "\2\20\u0100\3\2\2\2\22\u0104\3\2\2\2\24\u0120\3\2\2\2\26\u012a\3\2\2\2"+
-    "\30\u0134\3\2\2\2\32\u0143\3\2\2\2\34\u0145\3\2\2\2\36\u014b\3\2\2\2 "+
-    "\u014d\3\2\2\2\"\u0154\3\2\2\2$\u0166\3\2\2\2&\u0177\3\2\2\2(\u0187\3"+
-    "\2\2\2*\u01a2\3\2\2\2,\u01a4\3\2\2\2.\u01d7\3\2\2\2\60\u01e4\3\2\2\2\62"+
-    "\u0216\3\2\2\2\64\u0218\3\2\2\2\66\u0222\3\2\2\28\u0228\3\2\2\2:\u024e"+
-    "\3\2\2\2<\u0255\3\2\2\2>\u0257\3\2\2\2@\u0263\3\2\2\2B\u0265\3\2\2\2D"+
-    "\u0271\3\2\2\2F\u0273\3\2\2\2H\u029d\3\2\2\2J\u029f\3\2\2\2L\u02a1\3\2"+
-    "\2\2N\u02a3\3\2\2\2P\u02aa\3\2\2\2R\u02b1\3\2\2\2T\u02bf\3\2\2\2V\u02c3"+
-    "\3\2\2\2X\u02c8\3\2\2\2Z\u02cc\3\2\2\2\\\u02ce\3\2\2\2^\u02d0\3\2\2\2"+
-    "`a\5\6\4\2ab\7\2\2\3b\3\3\2\2\2cd\5,\27\2de\7\2\2\3e\5\3\2\2\2f\u00d4"+
-    "\5\b\5\2gu\7\33\2\2hq\7\3\2\2ij\78\2\2jp\t\2\2\2kl\7\36\2\2lp\t\3\2\2"+
-    "mn\7G\2\2np\5L\'\2oi\3\2\2\2ok\3\2\2\2om\3\2\2\2ps\3\2\2\2qo\3\2\2\2q"+
-    "r\3\2\2\2rt\3\2\2\2sq\3\2\2\2tv\7\4\2\2uh\3\2\2\2uv\3\2\2\2vw\3\2\2\2"+
-    "w\u00d4\5\6\4\2x\u0084\7\24\2\2y\u0080\7\3\2\2z{\78\2\2{\177\t\4\2\2|"+
-    "}\7\36\2\2}\177\t\3\2\2~z\3\2\2\2~|\3\2\2\2\177\u0082\3\2\2\2\u0080~\3"+
-    "\2\2\2\u0080\u0081\3\2\2\2\u0081\u0083\3\2\2\2\u0082\u0080\3\2\2\2\u0083"+
-    "\u0085\7\4\2\2\u0084y\3\2\2\2\u0084\u0085\3\2\2\2\u0085\u0086\3\2\2\2"+
-    "\u0086\u00d4\5\6\4\2\u0087\u0088\7>\2\2\u0088\u008d\7A\2\2\u0089\u008b"+
-    "\7*\2\2\u008a\u0089\3\2\2\2\u008a\u008b\3\2\2\2\u008b\u008c\3\2\2\2\u008c"+
-    "\u008e\5\64\33\2\u008d\u008a\3\2\2\2\u008d\u008e\3\2\2\2\u008e\u00d4\3"+
-    "\2\2\2\u008f\u0090\7>\2\2\u0090\u0091\7\23\2\2\u0091\u0092\t\5\2\2\u0092"+
-    "\u00d4\5T+\2\u0093\u0094\t\6\2\2\u0094\u00d4\5T+\2\u0095\u0096\7>\2\2"+
-    "\u0096\u009b\7!\2\2\u0097\u0099\7*\2\2\u0098\u0097\3\2\2\2\u0098\u0099"+
-    "\3\2\2\2\u0099\u009a\3\2\2\2\u009a\u009c\5\64\33\2\u009b\u0098\3\2\2\2"+
-    "\u009b\u009c\3\2\2\2\u009c\u00d4\3\2\2\2\u009d\u009e\7>\2\2\u009e\u00d4"+
-    "\7<\2\2\u009f\u00a0\7?\2\2\u00a0\u00d4\7\22\2\2\u00a1\u00a2\7?\2\2\u00a2"+
-    "\u00a8\7A\2\2\u00a3\u00a5\7\21\2\2\u00a4\u00a6\7*\2\2\u00a5\u00a4\3\2"+
-    "\2\2\u00a5\u00a6\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00a9\5\64\33\2\u00a8"+
-    "\u00a3\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\u00ae\3\2\2\2\u00aa\u00ac\7*"+
-    "\2\2\u00ab\u00aa\3\2\2\2\u00ab\u00ac\3\2\2\2\u00ac\u00ad\3\2\2\2\u00ad"+
-    "\u00af\5\64\33\2\u00ae\u00ab\3\2\2\2\u00ae\u00af\3\2\2\2\u00af\u00b9\3"+
-    "\2\2\2\u00b0\u00b1\7D\2\2\u00b1\u00b6\5\\/\2\u00b2\u00b3\7\5\2\2\u00b3"+
-    "\u00b5\5\\/\2\u00b4\u00b2\3\2\2\2\u00b5\u00b8\3\2\2\2\u00b6\u00b4\3\2"+
-    "\2\2\u00b6\u00b7\3\2\2\2\u00b7\u00ba\3\2\2\2\u00b8\u00b6\3\2\2\2\u00b9"+
-    "\u00b0\3\2\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00d4\3\2\2\2\u00bb\u00bc\7?"+
-    "\2\2\u00bc\u00bf\7\23\2\2\u00bd\u00be\7\21\2\2\u00be\u00c0\5\\/\2\u00bf"+
-    "\u00bd\3\2\2\2\u00bf\u00c0\3\2\2\2\u00c0\u00c6\3\2\2\2\u00c1\u00c3\7@"+
-    "\2\2\u00c2\u00c4\7*\2\2\u00c3\u00c2\3\2\2\2\u00c3\u00c4\3\2\2\2\u00c4"+
-    "\u00c5\3\2\2\2\u00c5\u00c7\5\64\33\2\u00c6\u00c1\3\2\2\2\u00c6\u00c7\3"+
-    "\2\2\2\u00c7\u00cc\3\2\2\2\u00c8\u00ca\7*\2\2\u00c9\u00c8\3\2\2\2\u00c9"+
-    "\u00ca\3\2\2\2\u00ca\u00cb\3\2\2\2\u00cb\u00cd\5\64\33\2\u00cc\u00c9\3"+
-    "\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\u00d4\3\2\2\2\u00ce\u00cf\7?\2\2\u00cf"+
-    "\u00d4\7E\2\2\u00d0\u00d1\7?\2\2\u00d1\u00d2\7@\2\2\u00d2\u00d4\7E\2\2"+
-    "\u00d3f\3\2\2\2\u00d3g\3\2\2\2\u00d3x\3\2\2\2\u00d3\u0087\3\2\2\2\u00d3"+
-    "\u008f\3\2\2\2\u00d3\u0093\3\2\2\2\u00d3\u0095\3\2\2\2\u00d3\u009d\3\2"+
-    "\2\2\u00d3\u009f\3\2\2\2\u00d3\u00a1\3\2\2\2\u00d3\u00bb\3\2\2\2\u00d3"+
-    "\u00ce\3\2\2\2\u00d3\u00d0\3\2\2\2\u00d4\7\3\2\2\2\u00d5\u00d6\7I\2\2"+
-    "\u00d6\u00db\5\34\17\2\u00d7\u00d8\7\5\2\2\u00d8\u00da\5\34\17\2\u00d9"+
-    "\u00d7\3\2\2\2\u00da\u00dd\3\2\2\2\u00db\u00d9\3\2\2\2\u00db\u00dc\3\2"+
-    "\2\2\u00dc\u00df\3\2\2\2\u00dd\u00db\3\2\2\2\u00de\u00d5\3\2\2\2\u00de"+
-    "\u00df\3\2\2\2\u00df\u00e0\3\2\2\2\u00e0\u00e1\5\n\6\2\u00e1\t\3\2\2\2"+
-    "\u00e2\u00ed\5\16\b\2\u00e3\u00e4\7\64\2\2\u00e4\u00e5\7\17\2\2\u00e5"+
-    "\u00ea\5\20\t\2\u00e6\u00e7\7\5\2\2\u00e7\u00e9\5\20\t\2\u00e8\u00e6\3"+
-    "\2\2\2\u00e9\u00ec\3\2\2\2\u00ea\u00e8\3\2\2\2\u00ea\u00eb\3\2\2\2\u00eb"+
-    "\u00ee\3\2\2\2\u00ec\u00ea\3\2\2\2\u00ed\u00e3\3\2\2\2\u00ed\u00ee\3\2"+
-    "\2\2\u00ee\u00f0\3\2\2\2\u00ef\u00f1\5\f\7\2\u00f0\u00ef\3\2\2\2\u00f0"+
-    "\u00f1\3\2\2\2\u00f1\13\3\2\2\2\u00f2\u00f3\7+\2\2\u00f3\u00f8\t\7\2\2"+
-    "\u00f4\u00f5\7L\2\2\u00f5\u00f6\t\7\2\2\u00f6\u00f8\7Q\2\2\u00f7\u00f2"+
-    "\3\2\2\2\u00f7\u00f4\3\2\2\2\u00f8\r\3\2\2\2\u00f9\u00ff\5\22\n\2\u00fa"+
-    "\u00fb\7\3\2\2\u00fb\u00fc\5\n\6\2\u00fc\u00fd\7\4\2\2\u00fd\u00ff\3\2"+
-    "\2\2\u00fe\u00f9\3\2\2\2\u00fe\u00fa\3\2\2\2\u00ff\17\3\2\2\2\u0100\u0102"+
-    "\5,\27\2\u0101\u0103\t\b\2\2\u0102\u0101\3\2\2\2\u0102\u0103\3\2\2\2\u0103"+
-    "\21\3\2\2\2\u0104\u0106\7=\2\2\u0105\u0107\5\36\20\2\u0106\u0105\3\2\2"+
-    "\2\u0106\u0107\3\2\2\2\u0107\u0108\3\2\2\2\u0108\u010d\5 \21\2\u0109\u010a"+
-    "\7\5\2\2\u010a\u010c\5 \21\2\u010b\u0109\3\2\2\2\u010c\u010f\3\2\2\2\u010d"+
-    "\u010b\3\2\2\2\u010d\u010e\3\2\2\2\u010e\u0111\3\2\2\2\u010f\u010d\3\2"+
-    "\2\2\u0110\u0112\5\24\13\2\u0111\u0110\3\2\2\2\u0111\u0112\3\2\2\2\u0112"+
-    "\u0115\3\2\2\2\u0113\u0114\7H\2\2\u0114\u0116\5.\30\2\u0115\u0113\3\2"+
-    "\2\2\u0115\u0116\3\2\2\2\u0116\u011a\3\2\2\2\u0117\u0118\7#\2\2\u0118"+
-    "\u0119\7\17\2\2\u0119\u011b\5\26\f\2\u011a\u0117\3\2\2\2\u011a\u011b\3"+
-    "\2\2\2\u011b\u011e\3\2\2\2\u011c\u011d\7$\2\2\u011d\u011f\5.\30\2\u011e"+
-    "\u011c\3\2\2\2\u011e\u011f\3\2\2\2\u011f\23\3\2\2\2\u0120\u0121\7\37\2"+
-    "\2\u0121\u0126\5\"\22\2\u0122\u0123\7\5\2\2\u0123\u0125\5\"\22\2\u0124"+
-    "\u0122\3\2\2\2\u0125\u0128\3\2\2\2\u0126\u0124\3\2\2\2\u0126\u0127\3\2"+
-    "\2\2\u0127\25\3\2\2\2\u0128\u0126\3\2\2\2\u0129\u012b\5\36\20\2\u012a"+
-    "\u0129\3\2\2\2\u012a\u012b\3\2\2\2\u012b\u012c\3\2\2\2\u012c\u0131\5\30"+
-    "\r\2\u012d\u012e\7\5\2\2\u012e\u0130\5\30\r\2\u012f\u012d\3\2\2\2\u0130"+
-    "\u0133\3\2\2\2\u0131\u012f\3\2\2\2\u0131\u0132\3\2\2\2\u0132\27\3\2\2"+
-    "\2\u0133\u0131\3\2\2\2\u0134\u0135\5\32\16\2\u0135\31\3\2\2\2\u0136\u013f"+
-    "\7\3\2\2\u0137\u013c\5,\27\2\u0138\u0139\7\5\2\2\u0139\u013b\5,\27\2\u013a"+
-    "\u0138\3\2\2\2\u013b\u013e\3\2\2\2\u013c\u013a\3\2\2\2\u013c\u013d\3\2"+
-    "\2\2\u013d\u0140\3\2\2\2\u013e\u013c\3\2\2\2\u013f\u0137\3\2\2\2\u013f"+
-    "\u0140\3\2\2\2\u0140\u0141\3\2\2\2\u0141\u0144\7\4\2\2\u0142\u0144\5,"+
-    "\27\2\u0143\u0136\3\2\2\2\u0143\u0142\3\2\2\2\u0144\33\3\2\2\2\u0145\u0146"+
-    "\5R*\2\u0146\u0147\7\f\2\2\u0147\u0148\7\3\2\2\u0148\u0149\5\n\6\2\u0149"+
-    "\u014a\7\4\2\2\u014a\35\3\2\2\2\u014b\u014c\t\t\2\2\u014c\37\3\2\2\2\u014d"+
-    "\u0152\5,\27\2\u014e\u0150\7\f\2\2\u014f\u014e\3\2\2\2\u014f\u0150\3\2"+
-    "\2\2\u0150\u0151\3\2\2\2\u0151\u0153\5R*\2\u0152\u014f\3\2\2\2\u0152\u0153"+
-    "\3\2\2\2\u0153!\3\2\2\2\u0154\u0158\5*\26\2\u0155\u0157\5$\23\2\u0156"+
-    "\u0155\3\2\2\2\u0157\u015a\3\2\2\2\u0158\u0156\3\2\2\2\u0158\u0159\3\2"+
-    "\2\2\u0159#\3\2\2\2\u015a\u0158\3\2\2\2\u015b\u015c\5&\24\2\u015c\u015d"+
-    "\7(\2\2\u015d\u015f\5*\26\2\u015e\u0160\5(\25\2\u015f\u015e\3\2\2\2\u015f"+
-    "\u0160\3\2\2\2\u0160\u0167\3\2\2\2\u0161\u0162\7.\2\2\u0162\u0163\5&\24"+
-    "\2\u0163\u0164\7(\2\2\u0164\u0165\5*\26\2\u0165\u0167\3\2\2\2\u0166\u015b"+
-    "\3\2\2\2\u0166\u0161\3\2\2\2\u0167%\3\2\2\2\u0168\u016a\7&\2\2\u0169\u0168"+
-    "\3\2\2\2\u0169\u016a\3\2\2\2\u016a\u0178\3\2\2\2\u016b\u016d\7)\2\2\u016c"+
-    "\u016e\7\65\2\2\u016d\u016c\3\2\2\2\u016d\u016e\3\2\2\2\u016e\u0178\3"+
-    "\2\2\2\u016f\u0171\79\2\2\u0170\u0172\7\65\2\2\u0171\u0170\3\2\2\2\u0171"+
-    "\u0172\3\2\2\2\u0172\u0178\3\2\2\2\u0173\u0175\7 \2\2\u0174\u0176\7\65"+
-    "\2\2\u0175\u0174\3\2\2\2\u0175\u0176\3\2\2\2\u0176\u0178\3\2\2\2\u0177"+
-    "\u0169\3\2\2\2\u0177\u016b\3\2\2\2\u0177\u016f\3\2\2\2\u0177\u0173\3\2"+
-    "\2\2\u0178\'\3\2\2\2\u0179\u017a\7\61\2\2\u017a\u0188\5.\30\2\u017b\u017c"+
-    "\7F\2\2\u017c\u017d\7\3\2\2\u017d\u0182\5R*\2\u017e\u017f\7\5\2\2\u017f"+
-    "\u0181\5R*\2\u0180\u017e\3\2\2\2\u0181\u0184\3\2\2\2\u0182\u0180\3\2\2"+
-    "\2\u0182\u0183\3\2\2\2\u0183\u0185\3\2\2\2\u0184\u0182\3\2\2\2\u0185\u0186"+
-    "\7\4\2\2\u0186\u0188\3\2\2\2\u0187\u0179\3\2\2\2\u0187\u017b\3\2\2\2\u0188"+
-    ")\3\2\2\2\u0189\u018e\5T+\2\u018a\u018c\7\f\2\2\u018b\u018a\3\2\2\2\u018b"+
-    "\u018c\3\2\2\2\u018c\u018d\3\2\2\2\u018d\u018f\5P)\2\u018e\u018b\3\2\2"+
-    "\2\u018e\u018f\3\2\2\2\u018f\u01a3\3\2\2\2\u0190\u0191\7\3\2\2\u0191\u0192"+
-    "\5\n\6\2\u0192\u0197\7\4\2\2\u0193\u0195\7\f\2\2\u0194\u0193\3\2\2\2\u0194"+
-    "\u0195\3\2\2\2\u0195\u0196\3\2\2\2\u0196\u0198\5P)\2\u0197\u0194\3\2\2"+
-    "\2\u0197\u0198\3\2\2\2\u0198\u01a3\3\2\2\2\u0199\u019a\7\3\2\2\u019a\u019b"+
-    "\5\"\22\2\u019b\u01a0\7\4\2\2\u019c\u019e\7\f\2\2\u019d\u019c\3\2\2\2"+
-    "\u019d\u019e\3\2\2\2\u019e\u019f\3\2\2\2\u019f\u01a1\5P)\2\u01a0\u019d"+
-    "\3\2\2\2\u01a0\u01a1\3\2\2\2\u01a1\u01a3\3\2\2\2\u01a2\u0189\3\2\2\2\u01a2"+
-    "\u0190\3\2\2\2\u01a2\u0199\3\2\2\2\u01a3+\3\2\2\2\u01a4\u01a5\5.\30\2"+
-    "\u01a5-\3\2\2\2\u01a6\u01a7\b\30\1\2\u01a7\u01a8\7/\2\2\u01a8\u01d8\5"+
-    ".\30\n\u01a9\u01aa\7\32\2\2\u01aa\u01ab\7\3\2\2\u01ab\u01ac\5\b\5\2\u01ac"+
-    "\u01ad\7\4\2\2\u01ad\u01d8\3\2\2\2\u01ae\u01af\7;\2\2\u01af\u01b0\7\3"+
-    "\2\2\u01b0\u01b5\5\\/\2\u01b1\u01b2\7\5\2\2\u01b2\u01b4\5\\/\2\u01b3\u01b1"+
-    "\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+
-    "\u01b8\3\2\2\2\u01b7\u01b5\3\2\2\2\u01b8\u01b9\7\4\2\2\u01b9\u01d8\3\2"+
-    "\2\2\u01ba\u01bb\7-\2\2\u01bb\u01bc\7\3\2\2\u01bc\u01bd\5P)\2\u01bd\u01be"+
-    "\7\5\2\2\u01be\u01c3\5\\/\2\u01bf\u01c0\7\5\2\2\u01c0\u01c2\5\\/\2\u01c1"+
-    "\u01bf\3\2\2\2\u01c2\u01c5\3\2\2\2\u01c3\u01c1\3\2\2\2\u01c3\u01c4\3\2"+
-    "\2\2\u01c4\u01c6\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c6\u01c7\7\4\2\2\u01c7"+
-    "\u01d8\3\2\2\2\u01c8\u01c9\7-\2\2\u01c9\u01ca\7\3\2\2\u01ca\u01cb\5\\"+
-    "/\2\u01cb\u01cc\7\5\2\2\u01cc\u01d1\5\\/\2\u01cd\u01ce\7\5\2\2\u01ce\u01d0"+
-    "\5\\/\2\u01cf\u01cd\3\2\2\2\u01d0\u01d3\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d1"+
-    "\u01d2\3\2\2\2\u01d2\u01d4\3\2\2\2\u01d3\u01d1\3\2\2\2\u01d4\u01d5\7\4"+
-    "\2\2\u01d5\u01d8\3\2\2\2\u01d6\u01d8\5\60\31\2\u01d7\u01a6\3\2\2\2\u01d7"+
-    "\u01a9\3\2\2\2\u01d7\u01ae\3\2\2\2\u01d7\u01ba\3\2\2\2\u01d7\u01c8\3\2"+
-    "\2\2\u01d7\u01d6\3\2\2\2\u01d8\u01e1\3\2\2\2\u01d9\u01da\f\4\2\2\u01da"+
-    "\u01db\7\n\2\2\u01db\u01e0\5.\30\5\u01dc\u01dd\f\3\2\2\u01dd\u01de\7\63"+
-    "\2\2\u01de\u01e0\5.\30\4\u01df\u01d9\3\2\2\2\u01df\u01dc\3\2\2\2\u01e0"+
-    "\u01e3\3\2\2\2\u01e1\u01df\3\2\2\2\u01e1\u01e2\3\2\2\2\u01e2/\3\2\2\2"+
-    "\u01e3\u01e1\3\2\2\2\u01e4\u01e6\58\35\2\u01e5\u01e7\5\62\32\2\u01e6\u01e5"+
-    "\3\2\2\2\u01e6\u01e7\3\2\2\2\u01e7\61\3\2\2\2\u01e8\u01ea\7/\2\2\u01e9"+
-    "\u01e8\3\2\2\2\u01e9\u01ea\3\2\2\2\u01ea\u01eb\3\2\2\2\u01eb\u01ec\7\16"+
-    "\2\2\u01ec\u01ed\58\35\2\u01ed\u01ee\7\n\2\2\u01ee\u01ef\58\35\2\u01ef"+
-    "\u0217\3\2\2\2\u01f0\u01f2\7/\2\2\u01f1\u01f0\3\2\2\2\u01f1\u01f2\3\2"+
-    "\2\2\u01f2\u01f3\3\2\2\2\u01f3\u01f4\7%\2\2\u01f4\u01f5\7\3\2\2\u01f5"+
-    "\u01fa\5,\27\2\u01f6\u01f7\7\5\2\2\u01f7\u01f9\5,\27\2\u01f8\u01f6\3\2"+
-    "\2\2\u01f9\u01fc\3\2\2\2\u01fa\u01f8\3\2\2\2\u01fa\u01fb\3\2\2\2\u01fb"+
-    "\u01fd\3\2\2\2\u01fc\u01fa\3\2\2\2\u01fd\u01fe\7\4\2\2\u01fe\u0217\3\2"+
-    "\2\2\u01ff\u0201\7/\2\2\u0200\u01ff\3\2\2\2\u0200\u0201\3\2\2\2\u0201"+
-    "\u0202\3\2\2\2\u0202\u0203\7%\2\2\u0203\u0204\7\3\2\2\u0204\u0205\5\b"+
-    "\5\2\u0205\u0206\7\4\2\2\u0206\u0217\3\2\2\2\u0207\u0209\7/\2\2\u0208"+
-    "\u0207\3\2\2\2\u0208\u0209\3\2\2\2\u0209\u020a\3\2\2\2\u020a\u020b\7*"+
-    "\2\2\u020b\u0217\5\64\33\2\u020c\u020e\7/\2\2\u020d\u020c\3\2\2\2\u020d"+
-    "\u020e\3\2\2\2\u020e\u020f\3\2\2\2\u020f\u0210\7:\2\2\u0210\u0217\5\\"+
-    "/\2\u0211\u0213\7\'\2\2\u0212\u0214\7/\2\2\u0213\u0212\3\2\2\2\u0213\u0214"+
-    "\3\2\2\2\u0214\u0215\3\2\2\2\u0215\u0217\7\60\2\2\u0216\u01e9\3\2\2\2"+
-    "\u0216\u01f1\3\2\2\2\u0216\u0200\3\2\2\2\u0216\u0208\3\2\2\2\u0216\u020d"+
-    "\3\2\2\2\u0216\u0211\3\2\2\2\u0217\63\3\2\2\2\u0218\u021a\5\\/\2\u0219"+
-    "\u021b\5\66\34\2\u021a\u0219\3\2\2\2\u021a\u021b\3\2\2\2\u021b\65\3\2"+
-    "\2\2\u021c\u021d\7\30\2\2\u021d\u0223\5\\/\2\u021e\u021f\7J\2\2\u021f"+
-    "\u0220\5\\/\2\u0220\u0221\7Q\2\2\u0221\u0223\3\2\2\2\u0222\u021c\3\2\2"+
-    "\2\u0222\u021e\3\2\2\2\u0223\67\3\2\2\2\u0224\u0225\b\35\1\2\u0225\u0229"+
-    "\5:\36\2\u0226\u0227\t\n\2\2\u0227\u0229\58\35\6\u0228\u0224\3\2\2\2\u0228"+
-    "\u0226\3\2\2\2\u0229\u0236\3\2\2\2\u022a\u022b\f\5\2\2\u022b\u022c\t\13"+
-    "\2\2\u022c\u0235\58\35\6\u022d\u022e\f\4\2\2\u022e\u022f\t\n\2\2\u022f"+
-    "\u0235\58\35\5\u0230\u0231\f\3\2\2\u0231\u0232\5J&\2\u0232\u0233\58\35"+
-    "\4\u0233\u0235\3\2\2\2\u0234\u022a\3\2\2\2\u0234\u022d\3\2\2\2\u0234\u0230"+
-    "\3\2\2\2\u0235\u0238\3\2\2\2\u0236\u0234\3\2\2\2\u0236\u0237\3\2\2\2\u0237"+
-    "9\3\2\2\2\u0238\u0236\3\2\2\2\u0239\u024f\5<\37\2\u023a\u024f\5@!\2\u023b"+
-    "\u024f\5H%\2\u023c\u024f\7Z\2\2\u023d\u023e\5P)\2\u023e\u023f\7^\2\2\u023f"+
-    "\u0241\3\2\2\2\u0240\u023d\3\2\2\2\u0240\u0241\3\2\2\2\u0241\u0242\3\2"+
-    "\2\2\u0242\u024f\7Z\2\2\u0243\u024f\5D#\2\u0244\u0245\7\3\2\2\u0245\u0246"+
-    "\5\b\5\2\u0246\u0247\7\4\2\2\u0247\u024f\3\2\2\2\u0248\u024f\5R*\2\u0249"+
-    "\u024f\5P)\2\u024a\u024b\7\3\2\2\u024b\u024c\5,\27\2\u024c\u024d\7\4\2"+
-    "\2\u024d\u024f\3\2\2\2\u024e\u0239\3\2\2\2\u024e\u023a\3\2\2\2\u024e\u023b"+
-    "\3\2\2\2\u024e\u023c\3\2\2\2\u024e\u0240\3\2\2\2\u024e\u0243\3\2\2\2\u024e"+
-    "\u0244\3\2\2\2\u024e\u0248\3\2\2\2\u024e\u0249\3\2\2\2\u024e\u024a\3\2"+
-    "\2\2\u024f;\3\2\2\2\u0250\u0256\5> \2\u0251\u0252\7K\2\2\u0252\u0253\5"+
-    "> \2\u0253\u0254\7Q\2\2\u0254\u0256\3\2\2\2\u0255\u0250\3\2\2\2\u0255"+
-    "\u0251\3\2\2\2\u0256=\3\2\2\2\u0257\u0258\7\20\2\2\u0258\u0259\7\3\2\2"+
-    "\u0259\u025a\5,\27\2\u025a\u025b\7\f\2\2\u025b\u025c\5N(\2\u025c\u025d"+
-    "\7\4\2\2\u025d?\3\2\2\2\u025e\u0264\5B\"\2\u025f\u0260\7K\2\2\u0260\u0261"+
-    "\5B\"\2\u0261\u0262\7Q\2\2\u0262\u0264\3\2\2\2\u0263\u025e\3\2\2\2\u0263"+
-    "\u025f\3\2\2\2\u0264A\3\2\2\2\u0265\u0266\7\34\2\2\u0266\u0267\7\3\2\2"+
-    "\u0267\u0268\5R*\2\u0268\u0269\7\37\2\2\u0269\u026a\58\35\2\u026a\u026b"+
-    "\7\4\2\2\u026bC\3\2\2\2\u026c\u0272\5F$\2\u026d\u026e\7K\2\2\u026e\u026f"+
-    "\5F$\2\u026f\u0270\7Q\2\2\u0270\u0272\3\2\2\2\u0271\u026c\3\2\2\2\u0271"+
-    "\u026d\3\2\2\2\u0272E\3\2\2\2\u0273\u0274\5R*\2\u0274\u0280\7\3\2\2\u0275"+
-    "\u0277\5\36\20\2\u0276\u0275\3\2\2\2\u0276\u0277\3\2\2\2\u0277\u0278\3"+
-    "\2\2\2\u0278\u027d\5,\27\2\u0279\u027a\7\5\2\2\u027a\u027c\5,\27\2\u027b"+
-    "\u0279\3\2\2\2\u027c\u027f\3\2\2\2\u027d\u027b\3\2\2\2\u027d\u027e\3\2"+
-    "\2\2\u027e\u0281\3\2\2\2\u027f\u027d\3\2\2\2\u0280\u0276\3\2\2\2\u0280"+
-    "\u0281\3\2\2\2\u0281\u0282\3\2\2\2\u0282\u0283\7\4\2\2\u0283G\3\2\2\2"+
-    "\u0284\u029e\7\60\2\2\u0285\u029e\5Z.\2\u0286\u029e\5L\'\2\u0287\u0289"+
-    "\7`\2\2\u0288\u0287\3\2\2\2\u0289\u028a\3\2\2\2\u028a\u0288\3\2\2\2\u028a"+
-    "\u028b\3\2\2\2\u028b\u029e\3\2\2\2\u028c\u029e\7_\2\2\u028d\u028e\7M\2"+
-    "\2\u028e\u028f\5\\/\2\u028f\u0290\7Q\2\2\u0290\u029e\3\2\2\2\u0291\u0292"+
-    "\7N\2\2\u0292\u0293\5\\/\2\u0293\u0294\7Q\2\2\u0294\u029e\3\2\2\2\u0295"+
-    "\u0296\7O\2\2\u0296\u0297\5\\/\2\u0297\u0298\7Q\2\2\u0298\u029e\3\2\2"+
-    "\2\u0299\u029a\7P\2\2\u029a\u029b\5\\/\2\u029b\u029c\7Q\2\2\u029c\u029e"+
-    "\3\2\2\2\u029d\u0284\3\2\2\2\u029d\u0285\3\2\2\2\u029d\u0286\3\2\2\2\u029d"+
-    "\u0288\3\2\2\2\u029d\u028c\3\2\2\2\u029d\u028d\3\2\2\2\u029d\u0291\3\2"+
-    "\2\2\u029d\u0295\3\2\2\2\u029d\u0299\3\2\2\2\u029eI\3\2\2\2\u029f\u02a0"+
-    "\t\f\2\2\u02a0K\3\2\2\2\u02a1\u02a2\t\r\2\2\u02a2M\3\2\2\2\u02a3\u02a4"+
-    "\5R*\2\u02a4O\3\2\2\2\u02a5\u02a6\5R*\2\u02a6\u02a7\7^\2\2\u02a7\u02a9"+
-    "\3\2\2\2\u02a8\u02a5\3\2\2\2\u02a9\u02ac\3\2\2\2\u02aa\u02a8\3\2\2\2\u02aa"+
-    "\u02ab\3\2\2\2\u02ab\u02ad\3\2\2\2\u02ac\u02aa\3\2\2\2\u02ad\u02ae\5R"+
-    "*\2\u02aeQ\3\2\2\2\u02af\u02b2\5V,\2\u02b0\u02b2\5X-\2\u02b1\u02af\3\2"+
-    "\2\2\u02b1\u02b0\3\2\2\2\u02b2S\3\2\2\2\u02b3\u02b4\5R*\2\u02b4\u02b5"+
-    "\7\6\2\2\u02b5\u02b7\3\2\2\2\u02b6\u02b3\3\2\2\2\u02b6\u02b7\3\2\2\2\u02b7"+
-    "\u02b8\3\2\2\2\u02b8\u02c0\7e\2\2\u02b9\u02ba\5R*\2\u02ba\u02bb\7\6\2"+
-    "\2\u02bb\u02bd\3\2\2\2\u02bc\u02b9\3\2\2\2\u02bc\u02bd\3\2\2\2\u02bd\u02be"+
-    "\3\2\2\2\u02be\u02c0\5R*\2\u02bf\u02b6\3\2\2\2\u02bf\u02bc\3\2\2\2\u02c0"+
-    "U\3\2\2\2\u02c1\u02c4\7f\2\2\u02c2\u02c4\7g\2\2\u02c3\u02c1\3\2\2\2\u02c3"+
-    "\u02c2\3\2\2\2\u02c4W\3\2\2\2\u02c5\u02c9\7c\2\2\u02c6\u02c9\5^\60\2\u02c7"+
-    "\u02c9\7d\2\2\u02c8\u02c5\3\2\2\2\u02c8\u02c6\3\2\2\2\u02c8\u02c7\3\2"+
-    "\2\2\u02c9Y\3\2\2\2\u02ca\u02cd\7b\2\2\u02cb\u02cd\7a\2\2\u02cc\u02ca"+
-    "\3\2\2\2\u02cc\u02cb\3\2\2\2\u02cd[\3\2\2\2\u02ce\u02cf\t\16\2\2\u02cf"+
-    "]\3\2\2\2\u02d0\u02d1\t\17\2\2\u02d1_\3\2\2\2eoqu~\u0080\u0084\u008a\u008d"+
-    "\u0098\u009b\u00a5\u00a8\u00ab\u00ae\u00b6\u00b9\u00bf\u00c3\u00c6\u00c9"+
-    "\u00cc\u00d3\u00db\u00de\u00ea\u00ed\u00f0\u00f7\u00fe\u0102\u0106\u010d"+
-    "\u0111\u0115\u011a\u011e\u0126\u012a\u0131\u013c\u013f\u0143\u014f\u0152"+
-    "\u0158\u015f\u0166\u0169\u016d\u0171\u0175\u0177\u0182\u0187\u018b\u018e"+
-    "\u0194\u0197\u019d\u01a0\u01a2\u01b5\u01c3\u01d1\u01d7\u01df\u01e1\u01e6"+
-    "\u01e9\u01f1\u01fa\u0200\u0208\u020d\u0213\u0216\u021a\u0222\u0228\u0234"+
-    "\u0236\u0240\u024e\u0255\u0263\u0271\u0276\u027d\u0280\u028a\u029d\u02aa"+
-    "\u02b1\u02b6\u02bc\u02bf\u02c3\u02c8\u02cc";
+    ",\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\3\2\3\2\3\2\3\3\3\3\3\3\3\4"+
+    "\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4r\n\4\f\4\16\4u\13\4\3\4\5\4x\n\4"+
+    "\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4\u0081\n\4\f\4\16\4\u0084\13\4\3\4\5\4"+
+    "\u0087\n\4\3\4\3\4\3\4\3\4\5\4\u008d\n\4\3\4\5\4\u0090\n\4\3\4\3\4\3\4"+
+    "\3\4\3\4\3\4\3\4\3\4\3\4\5\4\u009b\n\4\3\4\5\4\u009e\n\4\3\4\3\4\3\4\3"+
+    "\4\3\4\3\4\3\4\3\4\5\4\u00a8\n\4\3\4\5\4\u00ab\n\4\3\4\5\4\u00ae\n\4\3"+
+    "\4\5\4\u00b1\n\4\3\4\3\4\3\4\3\4\7\4\u00b7\n\4\f\4\16\4\u00ba\13\4\5\4"+
+    "\u00bc\n\4\3\4\3\4\3\4\3\4\5\4\u00c2\n\4\3\4\3\4\5\4\u00c6\n\4\3\4\5\4"+
+    "\u00c9\n\4\3\4\5\4\u00cc\n\4\3\4\5\4\u00cf\n\4\3\4\3\4\3\4\3\4\3\4\5\4"+
+    "\u00d6\n\4\3\5\3\5\3\5\3\5\7\5\u00dc\n\5\f\5\16\5\u00df\13\5\5\5\u00e1"+
+    "\n\5\3\5\3\5\3\6\3\6\3\6\3\6\3\6\3\6\7\6\u00eb\n\6\f\6\16\6\u00ee\13\6"+
+    "\5\6\u00f0\n\6\3\6\5\6\u00f3\n\6\3\7\3\7\3\7\3\7\3\7\5\7\u00fa\n\7\3\b"+
+    "\3\b\3\b\3\b\3\b\5\b\u0101\n\b\3\t\3\t\5\t\u0105\n\t\3\n\3\n\5\n\u0109"+
+    "\n\n\3\n\3\n\3\n\7\n\u010e\n\n\f\n\16\n\u0111\13\n\3\n\5\n\u0114\n\n\3"+
+    "\n\3\n\5\n\u0118\n\n\3\n\3\n\3\n\5\n\u011d\n\n\3\n\3\n\5\n\u0121\n\n\3"+
+    "\13\3\13\3\13\3\13\7\13\u0127\n\13\f\13\16\13\u012a\13\13\3\f\5\f\u012d"+
+    "\n\f\3\f\3\f\3\f\7\f\u0132\n\f\f\f\16\f\u0135\13\f\3\r\3\r\3\16\3\16\3"+
+    "\16\3\16\7\16\u013d\n\16\f\16\16\16\u0140\13\16\5\16\u0142\n\16\3\16\3"+
+    "\16\5\16\u0146\n\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\21\3\21"+
+    "\5\21\u0152\n\21\3\21\5\21\u0155\n\21\3\22\3\22\7\22\u0159\n\22\f\22\16"+
+    "\22\u015c\13\22\3\23\3\23\3\23\3\23\5\23\u0162\n\23\3\23\3\23\3\23\3\23"+
+    "\3\23\5\23\u0169\n\23\3\24\5\24\u016c\n\24\3\24\3\24\5\24\u0170\n\24\3"+
+    "\24\3\24\5\24\u0174\n\24\3\24\3\24\5\24\u0178\n\24\5\24\u017a\n\24\3\25"+
+    "\3\25\3\25\3\25\3\25\3\25\3\25\7\25\u0183\n\25\f\25\16\25\u0186\13\25"+
+    "\3\25\3\25\5\25\u018a\n\25\3\26\3\26\5\26\u018e\n\26\3\26\5\26\u0191\n"+
+    "\26\3\26\3\26\3\26\3\26\5\26\u0197\n\26\3\26\5\26\u019a\n\26\3\26\3\26"+
+    "\3\26\3\26\5\26\u01a0\n\26\3\26\5\26\u01a3\n\26\5\26\u01a5\n\26\3\27\3"+
+    "\27\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\7"+
+    "\30\u01b6\n\30\f\30\16\30\u01b9\13\30\3\30\3\30\3\30\3\30\3\30\3\30\3"+
+    "\30\3\30\3\30\7\30\u01c4\n\30\f\30\16\30\u01c7\13\30\3\30\3\30\3\30\3"+
+    "\30\3\30\3\30\3\30\3\30\3\30\7\30\u01d2\n\30\f\30\16\30\u01d5\13\30\3"+
+    "\30\3\30\3\30\5\30\u01da\n\30\3\30\3\30\3\30\3\30\3\30\3\30\7\30\u01e2"+
+    "\n\30\f\30\16\30\u01e5\13\30\3\31\3\31\5\31\u01e9\n\31\3\32\5\32\u01ec"+
+    "\n\32\3\32\3\32\3\32\3\32\3\32\3\32\5\32\u01f4\n\32\3\32\3\32\3\32\3\32"+
+    "\3\32\7\32\u01fb\n\32\f\32\16\32\u01fe\13\32\3\32\3\32\3\32\5\32\u0203"+
+    "\n\32\3\32\3\32\3\32\3\32\3\32\3\32\5\32\u020b\n\32\3\32\3\32\3\32\5\32"+
+    "\u0210\n\32\3\32\3\32\3\32\3\32\5\32\u0216\n\32\3\32\5\32\u0219\n\32\3"+
+    "\33\3\33\5\33\u021d\n\33\3\34\3\34\3\34\3\34\3\34\3\34\5\34\u0225\n\34"+
+    "\3\35\3\35\3\35\3\35\5\35\u022b\n\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35"+
+    "\3\35\3\35\3\35\7\35\u0237\n\35\f\35\16\35\u023a\13\35\3\36\3\36\3\36"+
+    "\3\36\3\36\3\36\3\36\5\36\u0243\n\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36"+
+    "\3\36\3\36\3\36\3\36\3\36\5\36\u0251\n\36\3\37\3\37\3\37\3\37\3\37\5\37"+
+    "\u0258\n\37\3 \3 \3 \3 \3 \3 \3 \3!\3!\3!\3!\3!\5!\u0266\n!\3\"\3\"\3"+
+    "\"\3\"\3\"\3\"\3\"\3#\3#\3#\3#\3#\5#\u0274\n#\3$\3$\3$\5$\u0279\n$\3$"+
+    "\3$\3$\7$\u027e\n$\f$\16$\u0281\13$\5$\u0283\n$\3$\3$\3%\3%\3%\5%\u028a"+
+    "\n%\3&\3&\3&\3&\6&\u0290\n&\r&\16&\u0291\3&\3&\3&\3&\3&\3&\3&\3&\3&\3"+
+    "&\3&\3&\3&\3&\3&\3&\3&\5&\u02a5\n&\3\'\3\'\3(\3(\3)\3)\3*\3*\3*\7*\u02b0"+
+    "\n*\f*\16*\u02b3\13*\3*\3*\3+\3+\5+\u02b9\n+\3,\3,\3,\5,\u02be\n,\3,\3"+
+    ",\3,\3,\5,\u02c4\n,\3,\5,\u02c7\n,\3-\3-\5-\u02cb\n-\3.\3.\3.\5.\u02d0"+
+    "\n.\3/\3/\5/\u02d4\n/\3\60\3\60\3\61\3\61\3\61\2\4.8\62\2\4\6\b\n\f\16"+
+    "\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<>@BDFHJLNPRTVXZ\\^`\2"+
+    "\20\b\2\7\7\t\t\31\31,,\62\62\66\66\4\2\"\"BB\4\2\t\t\62\62\4\2\37\37"+
+    "%%\3\2\25\26\4\2\7\7aa\4\2\r\r\25\25\4\2\7\7\27\27\3\2XY\3\2Z\\\3\2RW"+
+    "\4\2\35\35CC\3\2_`\20\2\b\t\22\24\31\31\33\33\36\36!\",,\62\62\668:<>"+
+    "?ABDEGG\u0336\2b\3\2\2\2\4e\3\2\2\2\6\u00d5\3\2\2\2\b\u00e0\3\2\2\2\n"+
+    "\u00e4\3\2\2\2\f\u00f9\3\2\2\2\16\u0100\3\2\2\2\20\u0102\3\2\2\2\22\u0106"+
+    "\3\2\2\2\24\u0122\3\2\2\2\26\u012c\3\2\2\2\30\u0136\3\2\2\2\32\u0145\3"+
+    "\2\2\2\34\u0147\3\2\2\2\36\u014d\3\2\2\2 \u014f\3\2\2\2\"\u0156\3\2\2"+
+    "\2$\u0168\3\2\2\2&\u0179\3\2\2\2(\u0189\3\2\2\2*\u01a4\3\2\2\2,\u01a6"+
+    "\3\2\2\2.\u01d9\3\2\2\2\60\u01e6\3\2\2\2\62\u0218\3\2\2\2\64\u021a\3\2"+
+    "\2\2\66\u0224\3\2\2\28\u022a\3\2\2\2:\u0250\3\2\2\2<\u0257\3\2\2\2>\u0259"+
+    "\3\2\2\2@\u0265\3\2\2\2B\u0267\3\2\2\2D\u0273\3\2\2\2F\u0275\3\2\2\2H"+
+    "\u0289\3\2\2\2J\u02a4\3\2\2\2L\u02a6\3\2\2\2N\u02a8\3\2\2\2P\u02aa\3\2"+
+    "\2\2R\u02b1\3\2\2\2T\u02b8\3\2\2\2V\u02c6\3\2\2\2X\u02ca\3\2\2\2Z\u02cf"+
+    "\3\2\2\2\\\u02d3\3\2\2\2^\u02d5\3\2\2\2`\u02d7\3\2\2\2bc\5\6\4\2cd\7\2"+
+    "\2\3d\3\3\2\2\2ef\5,\27\2fg\7\2\2\3g\5\3\2\2\2h\u00d6\5\b\5\2iw\7\33\2"+
+    "\2js\7\3\2\2kl\78\2\2lr\t\2\2\2mn\7\36\2\2nr\t\3\2\2op\7G\2\2pr\5N(\2"+
+    "qk\3\2\2\2qm\3\2\2\2qo\3\2\2\2ru\3\2\2\2sq\3\2\2\2st\3\2\2\2tv\3\2\2\2"+
+    "us\3\2\2\2vx\7\4\2\2wj\3\2\2\2wx\3\2\2\2xy\3\2\2\2y\u00d6\5\6\4\2z\u0086"+
+    "\7\24\2\2{\u0082\7\3\2\2|}\78\2\2}\u0081\t\4\2\2~\177\7\36\2\2\177\u0081"+
+    "\t\3\2\2\u0080|\3\2\2\2\u0080~\3\2\2\2\u0081\u0084\3\2\2\2\u0082\u0080"+
+    "\3\2\2\2\u0082\u0083\3\2\2\2\u0083\u0085\3\2\2\2\u0084\u0082\3\2\2\2\u0085"+
+    "\u0087\7\4\2\2\u0086{\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3\2\2\2"+
+    "\u0088\u00d6\5\6\4\2\u0089\u008a\7>\2\2\u008a\u008f\7A\2\2\u008b\u008d"+
+    "\7*\2\2\u008c\u008b\3\2\2\2\u008c\u008d\3\2\2\2\u008d\u008e\3\2\2\2\u008e"+
+    "\u0090\5\64\33\2\u008f\u008c\3\2\2\2\u008f\u0090\3\2\2\2\u0090\u00d6\3"+
+    "\2\2\2\u0091\u0092\7>\2\2\u0092\u0093\7\23\2\2\u0093\u0094\t\5\2\2\u0094"+
+    "\u00d6\5V,\2\u0095\u0096\t\6\2\2\u0096\u00d6\5V,\2\u0097\u0098\7>\2\2"+
+    "\u0098\u009d\7!\2\2\u0099\u009b\7*\2\2\u009a\u0099\3\2\2\2\u009a\u009b"+
+    "\3\2\2\2\u009b\u009c\3\2\2\2\u009c\u009e\5\64\33\2\u009d\u009a\3\2\2\2"+
+    "\u009d\u009e\3\2\2\2\u009e\u00d6\3\2\2\2\u009f\u00a0\7>\2\2\u00a0\u00d6"+
+    "\7<\2\2\u00a1\u00a2\7?\2\2\u00a2\u00d6\7\22\2\2\u00a3\u00a4\7?\2\2\u00a4"+
+    "\u00aa\7A\2\2\u00a5\u00a7\7\21\2\2\u00a6\u00a8\7*\2\2\u00a7\u00a6\3\2"+
+    "\2\2\u00a7\u00a8\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\u00ab\5\64\33\2\u00aa"+
+    "\u00a5\3\2\2\2\u00aa\u00ab\3\2\2\2\u00ab\u00b0\3\2\2\2\u00ac\u00ae\7*"+
+    "\2\2\u00ad\u00ac\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ae\u00af\3\2\2\2\u00af"+
+    "\u00b1\5\64\33\2\u00b0\u00ad\3\2\2\2\u00b0\u00b1\3\2\2\2\u00b1\u00bb\3"+
+    "\2\2\2\u00b2\u00b3\7D\2\2\u00b3\u00b8\5^\60\2\u00b4\u00b5\7\5\2\2\u00b5"+
+    "\u00b7\5^\60\2\u00b6\u00b4\3\2\2\2\u00b7\u00ba\3\2\2\2\u00b8\u00b6\3\2"+
+    "\2\2\u00b8\u00b9\3\2\2\2\u00b9\u00bc\3\2\2\2\u00ba\u00b8\3\2\2\2\u00bb"+
+    "\u00b2\3\2\2\2\u00bb\u00bc\3\2\2\2\u00bc\u00d6\3\2\2\2\u00bd\u00be\7?"+
+    "\2\2\u00be\u00c1\7\23\2\2\u00bf\u00c0\7\21\2\2\u00c0\u00c2\5^\60\2\u00c1"+
+    "\u00bf\3\2\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c8\3\2\2\2\u00c3\u00c5\7@"+
+    "\2\2\u00c4\u00c6\7*\2\2\u00c5\u00c4\3\2\2\2\u00c5\u00c6\3\2\2\2\u00c6"+
+    "\u00c7\3\2\2\2\u00c7\u00c9\5\64\33\2\u00c8\u00c3\3\2\2\2\u00c8\u00c9\3"+
+    "\2\2\2\u00c9\u00ce\3\2\2\2\u00ca\u00cc\7*\2\2\u00cb\u00ca\3\2\2\2\u00cb"+
+    "\u00cc\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\u00cf\5\64\33\2\u00ce\u00cb\3"+
+    "\2\2\2\u00ce\u00cf\3\2\2\2\u00cf\u00d6\3\2\2\2\u00d0\u00d1\7?\2\2\u00d1"+
+    "\u00d6\7E\2\2\u00d2\u00d3\7?\2\2\u00d3\u00d4\7@\2\2\u00d4\u00d6\7E\2\2"+
+    "\u00d5h\3\2\2\2\u00d5i\3\2\2\2\u00d5z\3\2\2\2\u00d5\u0089\3\2\2\2\u00d5"+
+    "\u0091\3\2\2\2\u00d5\u0095\3\2\2\2\u00d5\u0097\3\2\2\2\u00d5\u009f\3\2"+
+    "\2\2\u00d5\u00a1\3\2\2\2\u00d5\u00a3\3\2\2\2\u00d5\u00bd\3\2\2\2\u00d5"+
+    "\u00d0\3\2\2\2\u00d5\u00d2\3\2\2\2\u00d6\7\3\2\2\2\u00d7\u00d8\7I\2\2"+
+    "\u00d8\u00dd\5\34\17\2\u00d9\u00da\7\5\2\2\u00da\u00dc\5\34\17\2\u00db"+
+    "\u00d9\3\2\2\2\u00dc\u00df\3\2\2\2\u00dd\u00db\3\2\2\2\u00dd\u00de\3\2"+
+    "\2\2\u00de\u00e1\3\2\2\2\u00df\u00dd\3\2\2\2\u00e0\u00d7\3\2\2\2\u00e0"+
+    "\u00e1\3\2\2\2\u00e1\u00e2\3\2\2\2\u00e2\u00e3\5\n\6\2\u00e3\t\3\2\2\2"+
+    "\u00e4\u00ef\5\16\b\2\u00e5\u00e6\7\64\2\2\u00e6\u00e7\7\17\2\2\u00e7"+
+    "\u00ec\5\20\t\2\u00e8\u00e9\7\5\2\2\u00e9\u00eb\5\20\t\2\u00ea\u00e8\3"+
+    "\2\2\2\u00eb\u00ee\3\2\2\2\u00ec\u00ea\3\2\2\2\u00ec\u00ed\3\2\2\2\u00ed"+
+    "\u00f0\3\2\2\2\u00ee\u00ec\3\2\2\2\u00ef\u00e5\3\2\2\2\u00ef\u00f0\3\2"+
+    "\2\2\u00f0\u00f2\3\2\2\2\u00f1\u00f3\5\f\7\2\u00f2\u00f1\3\2\2\2\u00f2"+
+    "\u00f3\3\2\2\2\u00f3\13\3\2\2\2\u00f4\u00f5\7+\2\2\u00f5\u00fa\t\7\2\2"+
+    "\u00f6\u00f7\7L\2\2\u00f7\u00f8\t\7\2\2\u00f8\u00fa\7Q\2\2\u00f9\u00f4"+
+    "\3\2\2\2\u00f9\u00f6\3\2\2\2\u00fa\r\3\2\2\2\u00fb\u0101\5\22\n\2\u00fc"+
+    "\u00fd\7\3\2\2\u00fd\u00fe\5\n\6\2\u00fe\u00ff\7\4\2\2\u00ff\u0101\3\2"+
+    "\2\2\u0100\u00fb\3\2\2\2\u0100\u00fc\3\2\2\2\u0101\17\3\2\2\2\u0102\u0104"+
+    "\5,\27\2\u0103\u0105\t\b\2\2\u0104\u0103\3\2\2\2\u0104\u0105\3\2\2\2\u0105"+
+    "\21\3\2\2\2\u0106\u0108\7=\2\2\u0107\u0109\5\36\20\2\u0108\u0107\3\2\2"+
+    "\2\u0108\u0109\3\2\2\2\u0109\u010a\3\2\2\2\u010a\u010f\5 \21\2\u010b\u010c"+
+    "\7\5\2\2\u010c\u010e\5 \21\2\u010d\u010b\3\2\2\2\u010e\u0111\3\2\2\2\u010f"+
+    "\u010d\3\2\2\2\u010f\u0110\3\2\2\2\u0110\u0113\3\2\2\2\u0111\u010f\3\2"+
+    "\2\2\u0112\u0114\5\24\13\2\u0113\u0112\3\2\2\2\u0113\u0114\3\2\2\2\u0114"+
+    "\u0117\3\2\2\2\u0115\u0116\7H\2\2\u0116\u0118\5.\30\2\u0117\u0115\3\2"+
+    "\2\2\u0117\u0118\3\2\2\2\u0118\u011c\3\2\2\2\u0119\u011a\7#\2\2\u011a"+
+    "\u011b\7\17\2\2\u011b\u011d\5\26\f\2\u011c\u0119\3\2\2\2\u011c\u011d\3"+
+    "\2\2\2\u011d\u0120\3\2\2\2\u011e\u011f\7$\2\2\u011f\u0121\5.\30\2\u0120"+
+    "\u011e\3\2\2\2\u0120\u0121\3\2\2\2\u0121\23\3\2\2\2\u0122\u0123\7\37\2"+
+    "\2\u0123\u0128\5\"\22\2\u0124\u0125\7\5\2\2\u0125\u0127\5\"\22\2\u0126"+
+    "\u0124\3\2\2\2\u0127\u012a\3\2\2\2\u0128\u0126\3\2\2\2\u0128\u0129\3\2"+
+    "\2\2\u0129\25\3\2\2\2\u012a\u0128\3\2\2\2\u012b\u012d\5\36\20\2\u012c"+
+    "\u012b\3\2\2\2\u012c\u012d\3\2\2\2\u012d\u012e\3\2\2\2\u012e\u0133\5\30"+
+    "\r\2\u012f\u0130\7\5\2\2\u0130\u0132\5\30\r\2\u0131\u012f\3\2\2\2\u0132"+
+    "\u0135\3\2\2\2\u0133\u0131\3\2\2\2\u0133\u0134\3\2\2\2\u0134\27\3\2\2"+
+    "\2\u0135\u0133\3\2\2\2\u0136\u0137\5\32\16\2\u0137\31\3\2\2\2\u0138\u0141"+
+    "\7\3\2\2\u0139\u013e\5,\27\2\u013a\u013b\7\5\2\2\u013b\u013d\5,\27\2\u013c"+
+    "\u013a\3\2\2\2\u013d\u0140\3\2\2\2\u013e\u013c\3\2\2\2\u013e\u013f\3\2"+
+    "\2\2\u013f\u0142\3\2\2\2\u0140\u013e\3\2\2\2\u0141\u0139\3\2\2\2\u0141"+
+    "\u0142\3\2\2\2\u0142\u0143\3\2\2\2\u0143\u0146\7\4\2\2\u0144\u0146\5,"+
+    "\27\2\u0145\u0138\3\2\2\2\u0145\u0144\3\2\2\2\u0146\33\3\2\2\2\u0147\u0148"+
+    "\5T+\2\u0148\u0149\7\f\2\2\u0149\u014a\7\3\2\2\u014a\u014b\5\n\6\2\u014b"+
+    "\u014c\7\4\2\2\u014c\35\3\2\2\2\u014d\u014e\t\t\2\2\u014e\37\3\2\2\2\u014f"+
+    "\u0154\5,\27\2\u0150\u0152\7\f\2\2\u0151\u0150\3\2\2\2\u0151\u0152\3\2"+
+    "\2\2\u0152\u0153\3\2\2\2\u0153\u0155\5T+\2\u0154\u0151\3\2\2\2\u0154\u0155"+
+    "\3\2\2\2\u0155!\3\2\2\2\u0156\u015a\5*\26\2\u0157\u0159\5$\23\2\u0158"+
+    "\u0157\3\2\2\2\u0159\u015c\3\2\2\2\u015a\u0158\3\2\2\2\u015a\u015b\3\2"+
+    "\2\2\u015b#\3\2\2\2\u015c\u015a\3\2\2\2\u015d\u015e\5&\24\2\u015e\u015f"+
+    "\7(\2\2\u015f\u0161\5*\26\2\u0160\u0162\5(\25\2\u0161\u0160\3\2\2\2\u0161"+
+    "\u0162\3\2\2\2\u0162\u0169\3\2\2\2\u0163\u0164\7.\2\2\u0164\u0165\5&\24"+
+    "\2\u0165\u0166\7(\2\2\u0166\u0167\5*\26\2\u0167\u0169\3\2\2\2\u0168\u015d"+
+    "\3\2\2\2\u0168\u0163\3\2\2\2\u0169%\3\2\2\2\u016a\u016c\7&\2\2\u016b\u016a"+
+    "\3\2\2\2\u016b\u016c\3\2\2\2\u016c\u017a\3\2\2\2\u016d\u016f\7)\2\2\u016e"+
+    "\u0170\7\65\2\2\u016f\u016e\3\2\2\2\u016f\u0170\3\2\2\2\u0170\u017a\3"+
+    "\2\2\2\u0171\u0173\79\2\2\u0172\u0174\7\65\2\2\u0173\u0172\3\2\2\2\u0173"+
+    "\u0174\3\2\2\2\u0174\u017a\3\2\2\2\u0175\u0177\7 \2\2\u0176\u0178\7\65"+
+    "\2\2\u0177\u0176\3\2\2\2\u0177\u0178\3\2\2\2\u0178\u017a\3\2\2\2\u0179"+
+    "\u016b\3\2\2\2\u0179\u016d\3\2\2\2\u0179\u0171\3\2\2\2\u0179\u0175\3\2"+
+    "\2\2\u017a\'\3\2\2\2\u017b\u017c\7\61\2\2\u017c\u018a\5.\30\2\u017d\u017e"+
+    "\7F\2\2\u017e\u017f\7\3\2\2\u017f\u0184\5T+\2\u0180\u0181\7\5\2\2\u0181"+
+    "\u0183\5T+\2\u0182\u0180\3\2\2\2\u0183\u0186\3\2\2\2\u0184\u0182\3\2\2"+
+    "\2\u0184\u0185\3\2\2\2\u0185\u0187\3\2\2\2\u0186\u0184\3\2\2\2\u0187\u0188"+
+    "\7\4\2\2\u0188\u018a\3\2\2\2\u0189\u017b\3\2\2\2\u0189\u017d\3\2\2\2\u018a"+
+    ")\3\2\2\2\u018b\u0190\5V,\2\u018c\u018e\7\f\2\2\u018d\u018c\3\2\2\2\u018d"+
+    "\u018e\3\2\2\2\u018e\u018f\3\2\2\2\u018f\u0191\5R*\2\u0190\u018d\3\2\2"+
+    "\2\u0190\u0191\3\2\2\2\u0191\u01a5\3\2\2\2\u0192\u0193\7\3\2\2\u0193\u0194"+
+    "\5\n\6\2\u0194\u0199\7\4\2\2\u0195\u0197\7\f\2\2\u0196\u0195\3\2\2\2\u0196"+
+    "\u0197\3\2\2\2\u0197\u0198\3\2\2\2\u0198\u019a\5R*\2\u0199\u0196\3\2\2"+
+    "\2\u0199\u019a\3\2\2\2\u019a\u01a5\3\2\2\2\u019b\u019c\7\3\2\2\u019c\u019d"+
+    "\5\"\22\2\u019d\u01a2\7\4\2\2\u019e\u01a0\7\f\2\2\u019f\u019e\3\2\2\2"+
+    "\u019f\u01a0\3\2\2\2\u01a0\u01a1\3\2\2\2\u01a1\u01a3\5R*\2\u01a2\u019f"+
+    "\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3\u01a5\3\2\2\2\u01a4\u018b\3\2\2\2\u01a4"+
+    "\u0192\3\2\2\2\u01a4\u019b\3\2\2\2\u01a5+\3\2\2\2\u01a6\u01a7\5.\30\2"+
+    "\u01a7-\3\2\2\2\u01a8\u01a9\b\30\1\2\u01a9\u01aa\7/\2\2\u01aa\u01da\5"+
+    ".\30\n\u01ab\u01ac\7\32\2\2\u01ac\u01ad\7\3\2\2\u01ad\u01ae\5\b\5\2\u01ae"+
+    "\u01af\7\4\2\2\u01af\u01da\3\2\2\2\u01b0\u01b1\7;\2\2\u01b1\u01b2\7\3"+
+    "\2\2\u01b2\u01b7\5^\60\2\u01b3\u01b4\7\5\2\2\u01b4\u01b6\5^\60\2\u01b5"+
+    "\u01b3\3\2\2\2\u01b6\u01b9\3\2\2\2\u01b7\u01b5\3\2\2\2\u01b7\u01b8\3\2"+
+    "\2\2\u01b8\u01ba\3\2\2\2\u01b9\u01b7\3\2\2\2\u01ba\u01bb\7\4\2\2\u01bb"+
+    "\u01da\3\2\2\2\u01bc\u01bd\7-\2\2\u01bd\u01be\7\3\2\2\u01be\u01bf\5R*"+
+    "\2\u01bf\u01c0\7\5\2\2\u01c0\u01c5\5^\60\2\u01c1\u01c2\7\5\2\2\u01c2\u01c4"+
+    "\5^\60\2\u01c3\u01c1\3\2\2\2\u01c4\u01c7\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c5"+
+    "\u01c6\3\2\2\2\u01c6\u01c8\3\2\2\2\u01c7\u01c5\3\2\2\2\u01c8\u01c9\7\4"+
+    "\2\2\u01c9\u01da\3\2\2\2\u01ca\u01cb\7-\2\2\u01cb\u01cc\7\3\2\2\u01cc"+
+    "\u01cd\5^\60\2\u01cd\u01ce\7\5\2\2\u01ce\u01d3\5^\60\2\u01cf\u01d0\7\5"+
+    "\2\2\u01d0\u01d2\5^\60\2\u01d1\u01cf\3\2\2\2\u01d2\u01d5\3\2\2\2\u01d3"+
+    "\u01d1\3\2\2\2\u01d3\u01d4\3\2\2\2\u01d4\u01d6\3\2\2\2\u01d5\u01d3\3\2"+
+    "\2\2\u01d6\u01d7\7\4\2\2\u01d7\u01da\3\2\2\2\u01d8\u01da\5\60\31\2\u01d9"+
+    "\u01a8\3\2\2\2\u01d9\u01ab\3\2\2\2\u01d9\u01b0\3\2\2\2\u01d9\u01bc\3\2"+
+    "\2\2\u01d9\u01ca\3\2\2\2\u01d9\u01d8\3\2\2\2\u01da\u01e3\3\2\2\2\u01db"+
+    "\u01dc\f\4\2\2\u01dc\u01dd\7\n\2\2\u01dd\u01e2\5.\30\5\u01de\u01df\f\3"+
+    "\2\2\u01df\u01e0\7\63\2\2\u01e0\u01e2\5.\30\4\u01e1\u01db\3\2\2\2\u01e1"+
+    "\u01de\3\2\2\2\u01e2\u01e5\3\2\2\2\u01e3\u01e1\3\2\2\2\u01e3\u01e4\3\2"+
+    "\2\2\u01e4/\3\2\2\2\u01e5\u01e3\3\2\2\2\u01e6\u01e8\58\35\2\u01e7\u01e9"+
+    "\5\62\32\2\u01e8\u01e7\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\61\3\2\2\2\u01ea"+
+    "\u01ec\7/\2\2\u01eb\u01ea\3\2\2\2\u01eb\u01ec\3\2\2\2\u01ec\u01ed\3\2"+
+    "\2\2\u01ed\u01ee\7\16\2\2\u01ee\u01ef\58\35\2\u01ef\u01f0\7\n\2\2\u01f0"+
+    "\u01f1\58\35\2\u01f1\u0219\3\2\2\2\u01f2\u01f4\7/\2\2\u01f3\u01f2\3\2"+
+    "\2\2\u01f3\u01f4\3\2\2\2\u01f4\u01f5\3\2\2\2\u01f5\u01f6\7%\2\2\u01f6"+
+    "\u01f7\7\3\2\2\u01f7\u01fc\5,\27\2\u01f8\u01f9\7\5\2\2\u01f9\u01fb\5,"+
+    "\27\2\u01fa\u01f8\3\2\2\2\u01fb\u01fe\3\2\2\2\u01fc\u01fa\3\2\2\2\u01fc"+
+    "\u01fd\3\2\2\2\u01fd\u01ff\3\2\2\2\u01fe\u01fc\3\2\2\2\u01ff\u0200\7\4"+
+    "\2\2\u0200\u0219\3\2\2\2\u0201\u0203\7/\2\2\u0202\u0201\3\2\2\2\u0202"+
+    "\u0203\3\2\2\2\u0203\u0204\3\2\2\2\u0204\u0205\7%\2\2\u0205\u0206\7\3"+
+    "\2\2\u0206\u0207\5\b\5\2\u0207\u0208\7\4\2\2\u0208\u0219\3\2\2\2\u0209"+
+    "\u020b\7/\2\2\u020a\u0209\3\2\2\2\u020a\u020b\3\2\2\2\u020b\u020c\3\2"+
+    "\2\2\u020c\u020d\7*\2\2\u020d\u0219\5\64\33\2\u020e\u0210\7/\2\2\u020f"+
+    "\u020e\3\2\2\2\u020f\u0210\3\2\2\2\u0210\u0211\3\2\2\2\u0211\u0212\7:"+
+    "\2\2\u0212\u0219\5^\60\2\u0213\u0215\7\'\2\2\u0214\u0216\7/\2\2\u0215"+
+    "\u0214\3\2\2\2\u0215\u0216\3\2\2\2\u0216\u0217\3\2\2\2\u0217\u0219\7\60"+
+    "\2\2\u0218\u01eb\3\2\2\2\u0218\u01f3\3\2\2\2\u0218\u0202\3\2\2\2\u0218"+
+    "\u020a\3\2\2\2\u0218\u020f\3\2\2\2\u0218\u0213\3\2\2\2\u0219\63\3\2\2"+
+    "\2\u021a\u021c\5^\60\2\u021b\u021d\5\66\34\2\u021c\u021b\3\2\2\2\u021c"+
+    "\u021d\3\2\2\2\u021d\65\3\2\2\2\u021e\u021f\7\30\2\2\u021f\u0225\5^\60"+
+    "\2\u0220\u0221\7J\2\2\u0221\u0222\5^\60\2\u0222\u0223\7Q\2\2\u0223\u0225"+
+    "\3\2\2\2\u0224\u021e\3\2\2\2\u0224\u0220\3\2\2\2\u0225\67\3\2\2\2\u0226"+
+    "\u0227\b\35\1\2\u0227\u022b\5:\36\2\u0228\u0229\t\n\2\2\u0229\u022b\5"+
+    "8\35\6\u022a\u0226\3\2\2\2\u022a\u0228\3\2\2\2\u022b\u0238\3\2\2\2\u022c"+
+    "\u022d\f\5\2\2\u022d\u022e\t\13\2\2\u022e\u0237\58\35\6\u022f\u0230\f"+
+    "\4\2\2\u0230\u0231\t\n\2\2\u0231\u0237\58\35\5\u0232\u0233\f\3\2\2\u0233"+
+    "\u0234\5L\'\2\u0234\u0235\58\35\4\u0235\u0237\3\2\2\2\u0236\u022c\3\2"+
+    "\2\2\u0236\u022f\3\2\2\2\u0236\u0232\3\2\2\2\u0237\u023a\3\2\2\2\u0238"+
+    "\u0236\3\2\2\2\u0238\u0239\3\2\2\2\u02399\3\2\2\2\u023a\u0238\3\2\2\2"+
+    "\u023b\u0251\5<\37\2\u023c\u0251\5@!\2\u023d\u0251\5J&\2\u023e\u0251\7"+
+    "Z\2\2\u023f\u0240\5R*\2\u0240\u0241\7^\2\2\u0241\u0243\3\2\2\2\u0242\u023f"+
+    "\3\2\2\2\u0242\u0243\3\2\2\2\u0243\u0244\3\2\2\2\u0244\u0251\7Z\2\2\u0245"+
+    "\u0251\5D#\2\u0246\u0247\7\3\2\2\u0247\u0248\5\b\5\2\u0248\u0249\7\4\2"+
+    "\2\u0249\u0251\3\2\2\2\u024a\u0251\5T+\2\u024b\u0251\5R*\2\u024c\u024d"+
+    "\7\3\2\2\u024d\u024e\5,\27\2\u024e\u024f\7\4\2\2\u024f\u0251\3\2\2\2\u0250"+
+    "\u023b\3\2\2\2\u0250\u023c\3\2\2\2\u0250\u023d\3\2\2\2\u0250\u023e\3\2"+
+    "\2\2\u0250\u0242\3\2\2\2\u0250\u0245\3\2\2\2\u0250\u0246\3\2\2\2\u0250"+
+    "\u024a\3\2\2\2\u0250\u024b\3\2\2\2\u0250\u024c\3\2\2\2\u0251;\3\2\2\2"+
+    "\u0252\u0258\5> \2\u0253\u0254\7K\2\2\u0254\u0255\5> \2\u0255\u0256\7"+
+    "Q\2\2\u0256\u0258\3\2\2\2\u0257\u0252\3\2\2\2\u0257\u0253\3\2\2\2\u0258"+
+    "=\3\2\2\2\u0259\u025a\7\20\2\2\u025a\u025b\7\3\2\2\u025b\u025c\5,\27\2"+
+    "\u025c\u025d\7\f\2\2\u025d\u025e\5P)\2\u025e\u025f\7\4\2\2\u025f?\3\2"+
+    "\2\2\u0260\u0266\5B\"\2\u0261\u0262\7K\2\2\u0262\u0263\5B\"\2\u0263\u0264"+
+    "\7Q\2\2\u0264\u0266\3\2\2\2\u0265\u0260\3\2\2\2\u0265\u0261\3\2\2\2\u0266"+
+    "A\3\2\2\2\u0267\u0268\7\34\2\2\u0268\u0269\7\3\2\2\u0269\u026a\5T+\2\u026a"+
+    "\u026b\7\37\2\2\u026b\u026c\58\35\2\u026c\u026d\7\4\2\2\u026dC\3\2\2\2"+
+    "\u026e\u0274\5F$\2\u026f\u0270\7K\2\2\u0270\u0271\5F$\2\u0271\u0272\7"+
+    "Q\2\2\u0272\u0274\3\2\2\2\u0273\u026e\3\2\2\2\u0273\u026f\3\2\2\2\u0274"+
+    "E\3\2\2\2\u0275\u0276\5H%\2\u0276\u0282\7\3\2\2\u0277\u0279\5\36\20\2"+
+    "\u0278\u0277\3\2\2\2\u0278\u0279\3\2\2\2\u0279\u027a\3\2\2\2\u027a\u027f"+
+    "\5,\27\2\u027b\u027c\7\5\2\2\u027c\u027e\5,\27\2\u027d\u027b\3\2\2\2\u027e"+
+    "\u0281\3\2\2\2\u027f\u027d\3\2\2\2\u027f\u0280\3\2\2\2\u0280\u0283\3\2"+
+    "\2\2\u0281\u027f\3\2\2\2\u0282\u0278\3\2\2\2\u0282\u0283\3\2\2\2\u0283"+
+    "\u0284\3\2\2\2\u0284\u0285\7\4\2\2\u0285G\3\2\2\2\u0286\u028a\7)\2\2\u0287"+
+    "\u028a\79\2\2\u0288\u028a\5T+\2\u0289\u0286\3\2\2\2\u0289\u0287\3\2\2"+
+    "\2\u0289\u0288\3\2\2\2\u028aI\3\2\2\2\u028b\u02a5\7\60\2\2\u028c\u02a5"+
+    "\5\\/\2\u028d\u02a5\5N(\2\u028e\u0290\7`\2\2\u028f\u028e\3\2\2\2\u0290"+
+    "\u0291\3\2\2\2\u0291\u028f\3\2\2\2\u0291\u0292\3\2\2\2\u0292\u02a5\3\2"+
+    "\2\2\u0293\u02a5\7_\2\2\u0294\u0295\7M\2\2\u0295\u0296\5^\60\2\u0296\u0297"+
+    "\7Q\2\2\u0297\u02a5\3\2\2\2\u0298\u0299\7N\2\2\u0299\u029a\5^\60\2\u029a"+
+    "\u029b\7Q\2\2\u029b\u02a5\3\2\2\2\u029c\u029d\7O\2\2\u029d\u029e\5^\60"+
+    "\2\u029e\u029f\7Q\2\2\u029f\u02a5\3\2\2\2\u02a0\u02a1\7P\2\2\u02a1\u02a2"+
+    "\5^\60\2\u02a2\u02a3\7Q\2\2\u02a3\u02a5\3\2\2\2\u02a4\u028b\3\2\2\2\u02a4"+
+    "\u028c\3\2\2\2\u02a4\u028d\3\2\2\2\u02a4\u028f\3\2\2\2\u02a4\u0293\3\2"+
+    "\2\2\u02a4\u0294\3\2\2\2\u02a4\u0298\3\2\2\2\u02a4\u029c\3\2\2\2\u02a4"+
+    "\u02a0\3\2\2\2\u02a5K\3\2\2\2\u02a6\u02a7\t\f\2\2\u02a7M\3\2\2\2\u02a8"+
+    "\u02a9\t\r\2\2\u02a9O\3\2\2\2\u02aa\u02ab\5T+\2\u02abQ\3\2\2\2\u02ac\u02ad"+
+    "\5T+\2\u02ad\u02ae\7^\2\2\u02ae\u02b0\3\2\2\2\u02af\u02ac\3\2\2\2\u02b0"+
+    "\u02b3\3\2\2\2\u02b1\u02af\3\2\2\2\u02b1\u02b2\3\2\2\2\u02b2\u02b4\3\2"+
+    "\2\2\u02b3\u02b1\3\2\2\2\u02b4\u02b5\5T+\2\u02b5S\3\2\2\2\u02b6\u02b9"+
+    "\5X-\2\u02b7\u02b9\5Z.\2\u02b8\u02b6\3\2\2\2\u02b8\u02b7\3\2\2\2\u02b9"+
+    "U\3\2\2\2\u02ba\u02bb\5T+\2\u02bb\u02bc\7\6\2\2\u02bc\u02be\3\2\2\2\u02bd"+
+    "\u02ba\3\2\2\2\u02bd\u02be\3\2\2\2\u02be\u02bf\3\2\2\2\u02bf\u02c7\7e"+
+    "\2\2\u02c0\u02c1\5T+\2\u02c1\u02c2\7\6\2\2\u02c2\u02c4\3\2\2\2\u02c3\u02c0"+
+    "\3\2\2\2\u02c3\u02c4\3\2\2\2\u02c4\u02c5\3\2\2\2\u02c5\u02c7\5T+\2\u02c6"+
+    "\u02bd\3\2\2\2\u02c6\u02c3\3\2\2\2\u02c7W\3\2\2\2\u02c8\u02cb\7f\2\2\u02c9"+
+    "\u02cb\7g\2\2\u02ca\u02c8\3\2\2\2\u02ca\u02c9\3\2\2\2\u02cbY\3\2\2\2\u02cc"+
+    "\u02d0\7c\2\2\u02cd\u02d0\5`\61\2\u02ce\u02d0\7d\2\2\u02cf\u02cc\3\2\2"+
+    "\2\u02cf\u02cd\3\2\2\2\u02cf\u02ce\3\2\2\2\u02d0[\3\2\2\2\u02d1\u02d4"+
+    "\7b\2\2\u02d2\u02d4\7a\2\2\u02d3\u02d1\3\2\2\2\u02d3\u02d2\3\2\2\2\u02d4"+
+    "]\3\2\2\2\u02d5\u02d6\t\16\2\2\u02d6_\3\2\2\2\u02d7\u02d8\t\17\2\2\u02d8"+
+    "a\3\2\2\2fqsw\u0080\u0082\u0086\u008c\u008f\u009a\u009d\u00a7\u00aa\u00ad"+
+    "\u00b0\u00b8\u00bb\u00c1\u00c5\u00c8\u00cb\u00ce\u00d5\u00dd\u00e0\u00ec"+
+    "\u00ef\u00f2\u00f9\u0100\u0104\u0108\u010f\u0113\u0117\u011c\u0120\u0128"+
+    "\u012c\u0133\u013e\u0141\u0145\u0151\u0154\u015a\u0161\u0168\u016b\u016f"+
+    "\u0173\u0177\u0179\u0184\u0189\u018d\u0190\u0196\u0199\u019f\u01a2\u01a4"+
+    "\u01b7\u01c5\u01d3\u01d9\u01e1\u01e3\u01e8\u01eb\u01f3\u01fc\u0202\u020a"+
+    "\u020f\u0215\u0218\u021c\u0224\u022a\u0236\u0238\u0242\u0250\u0257\u0265"+
+    "\u0273\u0278\u027f\u0282\u0289\u0291\u02a4\u02b1\u02b8\u02bd\u02c3\u02c6"+
+    "\u02ca\u02cf\u02d3";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java
index 6745b3fa89b..b2ad5c8f770 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java
@@ -445,6 +445,12 @@ interface SqlBaseVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx);
+  /**
+   * Visit a parse tree produced by {@link SqlBaseParser#functionName}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitFunctionName(SqlBaseParser.FunctionNameContext ctx);
   /**
    * Visit a parse tree produced by the {@code nullLiteral}
    * labeled alternative in {@link SqlBaseParser#constant}.
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
index 3702939dd37..ecb5b83896e 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.sql.parser;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.expression.Expression;
 import org.elasticsearch.xpack.sql.expression.Literal;
+import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.Neg;
 import org.elasticsearch.xpack.sql.type.DataType;
 
@@ -15,6 +16,13 @@ public class ExpressionTests extends ESTestCase {
 
     private final SqlParser parser = new SqlParser();
 
+    public void testTokenFunctionName() throws Exception {
+        Expression lt = parser.createExpression("LEFT()");
+        assertEquals(UnresolvedFunction.class, lt.getClass());
+        UnresolvedFunction uf = (UnresolvedFunction) lt;
+        assertEquals("LEFT", uf.functionName());
+    }
+
     public void testLiteralLong() throws Exception {
         Expression lt = parser.createExpression(String.valueOf(Long.MAX_VALUE));
         assertEquals(Literal.class, lt.getClass());
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java
index f7f03e5e4b7..de9c6c56da0 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java
@@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class SqlParserTests extends ESTestCase {
+
     public void testSelectStar() {
         singleProjection(project(parseStatement("SELECT * FROM foo")), UnresolvedStar.class);
     }
@@ -44,6 +45,11 @@ public class SqlParserTests extends ESTestCase {
         assertEquals("SCORE", f.functionName());
     }
 
+    public void testSelectRightFunction() {
+        UnresolvedFunction f = singleProjection(project(parseStatement("SELECT RIGHT()")), UnresolvedFunction.class);
+        assertEquals("RIGHT", f.functionName());
+    }
+
     public void testOrderByField() {
         Order.OrderDirection dir = randomFrom(Order.OrderDirection.values());
         OrderBy ob = orderBy(parseStatement("SELECT * FROM foo ORDER BY bar" + stringForDirection(dir)));

From 14d7e2c7b2d3761a361edd720f98353f856936a4 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 16 Jul 2018 13:57:17 -0700
Subject: [PATCH 046/260] Revert "[test] disable packaging tests for suse
 boxes"

This reverts commit 30d6fd3ca74c44de27f8314fe7c070ac1d23819a.
---
 .../gradle/vagrant/VagrantTestPlugin.groovy          | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
index de3c0dfc328..d4d1d857e90 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -526,11 +526,7 @@ class VagrantTestPlugin implements Plugin<Project> {
                     project.gradle.removeListener(batsPackagingReproListener)
                 }
                 if (project.extensions.esvagrant.boxes.contains(box)) {
-                    // these tests are temporarily disabled for suse boxes while we debug an issue
-                    // https://github.com/elastic/elasticsearch/issues/30295
-                    if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
-                        packagingTest.dependsOn(batsPackagingTest)
-                    }
+                    packagingTest.dependsOn(batsPackagingTest)
                 }
             }
 
@@ -569,11 +565,7 @@ class VagrantTestPlugin implements Plugin<Project> {
                 project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                // these tests are temporarily disabled for suse boxes while we debug an issue
-                // https://github.com/elastic/elasticsearch/issues/30295
-                if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
-                    packagingTest.dependsOn(javaPackagingTest)
-                }
+                packagingTest.dependsOn(javaPackagingTest)
             }
 
             /*

From 791b9b147cb1763678500ca91cc28e7fa171f40d Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Mon, 16 Jul 2018 17:20:50 -0400
Subject: [PATCH 047/260] [Rollup] Add new capabilities endpoint for concrete
 rollup indices (#30401)

This introduces a new GetRollupIndexCaps API which allows the user to retrieve rollup capabilities of a specific rollup index (or index pattern). This is distinct from the existing RollupCaps endpoint.

- Multiple jobs can be stored in multiple indices and point to a single target data index pattern (logstash-*). The existing API finds capabilities/config of all jobs matching that data index pattern.
- One rollup index can hold data from multiple jobs, targeting multiple data index patterns. This new API finds the capabilities based on the concrete rollup indices.
---
 .../rollup/rollup-index-caps.asciidoc         | 161 ++++++++
 .../rollup/action/GetRollupCapsAction.java    |   2 +-
 .../action/GetRollupIndexCapsAction.java      | 195 ++++++++++
 .../elasticsearch/xpack/rollup/Rollup.java    |  33 +-
 .../action/TransportGetRollupCapsAction.java  |   7 +-
 .../TransportGetRollupIndexCapsAction.java    |  79 ++++
 .../rest/RestGetRollupIndexCapsAction.java    |  38 ++
 .../GetRollupCapsActionRequestTests.java      |   2 +-
 .../GetRollupIndexCapsActionRequestTests.java | 177 +++++++++
 .../xpack.rollup.get_rollup_index_caps.json   |  17 +
 .../test/rollup/get_rollup_caps.yml           |  13 +-
 .../test/rollup/get_rollup_index_caps.yml     | 363 ++++++++++++++++++
 12 files changed, 1066 insertions(+), 21 deletions(-)
 create mode 100644 x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java
 create mode 100644 x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
 create mode 100644 x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java
 create mode 100644 x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml

diff --git a/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc
new file mode 100644
index 00000000000..4636d9775e9
--- /dev/null
+++ b/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc
@@ -0,0 +1,161 @@
+[role="xpack"]
+[[rollup-get-rollup-index-caps]]
+=== Get Rollup Index Capabilities
+++++
+<titleabbrev>Get Rollup Index Caps</titleabbrev>
+++++
+
+experimental[]
+
+This API returns the rollup capabilities of all jobs inside of a rollup index (e.g. the index where rollup data is stored).
+A single rollup index may store the data for multiple rollup jobs, and may have a variety of capabilities depending on those jobs.
+
+This API will allow you to determine:
+
+1. What jobs are stored in an index (or indices specified via a pattern)?
+2. What target indices were rolled up, what fields were used in those rollups and what aggregations can be performed on each job?
+
+==== Request
+
+`GET {index}/_xpack/rollup/data`
+
+//===== Description
+
+==== Path Parameters
+
+`index`::
+  (string) Index or index-pattern of concrete rollup indices to check for capabilities.
+
+
+
+==== Request Body
+
+There is no request body for the Get Jobs API.
+
+==== Authorization
+
+You must have `monitor`, `monitor_rollup`, `manage` or `manage_rollup` cluster privileges to use this API.
+For more information, see
+{xpack-ref}/security-privileges.html[Security Privileges].
+
+==== Examples
+
+Imagine we have an index named `sensor-1` full of raw data.  We know that the data will grow over time, so there
+will be a `sensor-2`, `sensor-3`, etc.  Let's create a Rollup job, which stores it's data in `sensor_rollup`:
+
+[source,js]
+--------------------------------------------------
+PUT _xpack/rollup/job/sensor
+{
+    "index_pattern": "sensor-*",
+    "rollup_index": "sensor_rollup",
+    "cron": "*/30 * * * * ?",
+    "page_size" :1000,
+    "groups" : {
+      "date_histogram": {
+        "field": "timestamp",
+        "interval": "1h",
+        "delay": "7d"
+      },
+      "terms": {
+        "fields": ["node"]
+      }
+    },
+    "metrics": [
+        {
+            "field": "temperature",
+            "metrics": ["min", "max", "sum"]
+        },
+        {
+            "field": "voltage",
+            "metrics": ["avg"]
+        }
+    ]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sensor_index]
+
+If at a later date, we'd like to determine what jobs and capabilities were stored in the `sensor_rollup` index, we can use the Get Rollup
+Index API:
+
+[source,js]
+--------------------------------------------------
+GET /sensor_rollup/_xpack/rollup/data
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Note how we are requesting the concrete rollup index name (`sensor_rollup`) as the first part of the URL.
+This  will yield the following response:
+
+[source,js]
+----
+{
+  "sensor_rollup" : {
+    "rollup_jobs" : [
+      {
+        "job_id" : "sensor",
+        "rollup_index" : "sensor_rollup",
+        "index_pattern" : "sensor-*",
+        "fields" : {
+          "node" : [
+            {
+              "agg" : "terms"
+            }
+          ],
+          "temperature" : [
+            {
+              "agg" : "min"
+            },
+            {
+              "agg" : "max"
+            },
+            {
+              "agg" : "sum"
+            }
+          ],
+          "timestamp" : [
+            {
+              "agg" : "date_histogram",
+              "time_zone" : "UTC",
+              "interval" : "1h",
+              "delay": "7d"
+            }
+          ],
+          "voltage" : [
+            {
+              "agg" : "avg"
+            }
+          ]
+        }
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE
+
+
+The response that is returned contains information that is similar to the original Rollup configuration, but formatted
+differently.  First, there are some house-keeping details: the Rollup job's ID, the index that holds the rolled data,
+the index pattern that the job was targeting.
+
+Next it shows a list of fields that contain data eligible for rollup searches.  Here we see four fields: `node`, `temperature`,
+`timestamp` and `voltage`.  Each of these fields list the aggregations that are possible.  For example, you can use a min, max
+or sum aggregation on the `temperature` field, but only a `date_histogram` on `timestamp`.
+
+Note that the `rollup_jobs` element is an array; there can be multiple, independent jobs configured for a single index
+or index pattern.  Each of these jobs may have different configurations, so the API returns a list of all the various
+configurations available.
+
+
+Like other APIs that interact with indices, you can specify index patterns instead of explicit indices:
+
+[source,js]
+--------------------------------------------------
+GET /*_rollup/_xpack/rollup/data
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java
index ea98c2f4628..128874a6c8c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java
@@ -139,7 +139,7 @@ public class GetRollupCapsAction extends Action<GetRollupCapsAction.Response> {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             for (Map.Entry<String, RollableIndexCaps> entry : jobs.entrySet()) {
-               entry.getValue().toXContent(builder, params);
+                entry.getValue().toXContent(builder, params);
             }
             builder.endObject();
             return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java
new file mode 100644
index 00000000000..4f95919c498
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.rollup.action;
+
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.rollup.RollupField;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+
+public class GetRollupIndexCapsAction extends Action<GetRollupIndexCapsAction.Response> {
+
+    public static final GetRollupIndexCapsAction INSTANCE = new GetRollupIndexCapsAction();
+    public static final String NAME = "indices:data/read/xpack/rollup/get/index/caps";
+    public static final ParseField CONFIG = new ParseField("config");
+    public static final ParseField STATUS = new ParseField("status");
+    private static final ParseField INDICES_OPTIONS = new ParseField("indices_options");
+
+    private GetRollupIndexCapsAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Request extends ActionRequest implements IndicesRequest.Replaceable, ToXContent {
+        private String[] indices;
+        private IndicesOptions options;
+
+        public Request(String[] indices) {
+            this(indices, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED);
+        }
+
+        public Request(String[] indices, IndicesOptions options) {
+            this.indices = indices;
+            this.options = options;
+        }
+
+        public Request() {}
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            return options;
+        }
+
+        @Override
+        public String[] indices() {
+            return indices;
+        }
+
+        @Override
+        public IndicesRequest indices(String... indices) {
+            Objects.requireNonNull(indices, "indices must not be null");
+            for (String index : indices) {
+                Objects.requireNonNull(index, "index must not be null");
+            }
+            this.indices = indices;
+            return this;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            this.indices = in.readStringArray();
+            this.options = IndicesOptions.readIndicesOptions(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(indices);
+            options.writeIndicesOptions(out);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.array(RollupField.ID.getPreferredName(), indices);
+            builder.field(INDICES_OPTIONS.getPreferredName(), options);
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(Arrays.hashCode(indices), options);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+            return Arrays.equals(indices, other.indices)
+                && Objects.equals(options, other.options);
+        }
+    }
+
+    public static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
+
+        protected RequestBuilder(ElasticsearchClient client, GetRollupIndexCapsAction action) {
+            super(client, action, new Request());
+        }
+    }
+
+    public static class Response extends ActionResponse implements Writeable, ToXContentObject {
+
+        private Map<String, RollableIndexCaps> jobs = Collections.emptyMap();
+
+        public Response() {
+
+        }
+
+        public Response(Map<String, RollableIndexCaps> jobs) {
+            this.jobs = Objects.requireNonNull(jobs);
+        }
+
+        Response(StreamInput in) throws IOException {
+            jobs = in.readMap(StreamInput::readString, RollableIndexCaps::new);
+        }
+
+        public Map<String, RollableIndexCaps> getJobs() {
+            return jobs;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeMap(jobs, StreamOutput::writeString, (out1, value) -> value.writeTo(out1));
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            for (Map.Entry<String, RollableIndexCaps> entry : jobs.entrySet()) {
+                entry.getValue().toXContent(builder, params);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(jobs);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Response other = (Response) obj;
+            return Objects.equals(jobs, other.jobs);
+        }
+
+        @Override
+        public final String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
index cc24a0b4ab9..546103df5dd 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
@@ -38,6 +38,7 @@ import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.action.DeleteRollupJobAction;
 import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction;
+import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
 import org.elasticsearch.xpack.core.rollup.action.GetRollupJobsAction;
 import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction;
 import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
@@ -47,6 +48,7 @@ import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 import org.elasticsearch.xpack.rollup.action.TransportDeleteRollupJobAction;
 import org.elasticsearch.xpack.rollup.action.TransportGetRollupCapsAction;
+import org.elasticsearch.xpack.rollup.action.TransportGetRollupIndexCapsAction;
 import org.elasticsearch.xpack.rollup.action.TransportGetRollupJobAction;
 import org.elasticsearch.xpack.rollup.action.TransportPutRollupJobAction;
 import org.elasticsearch.xpack.rollup.action.TransportRollupSearchAction;
@@ -55,6 +57,7 @@ import org.elasticsearch.xpack.rollup.action.TransportStopRollupAction;
 import org.elasticsearch.xpack.rollup.job.RollupJobTask;
 import org.elasticsearch.xpack.rollup.rest.RestDeleteRollupJobAction;
 import org.elasticsearch.xpack.rollup.rest.RestGetRollupCapsAction;
+import org.elasticsearch.xpack.rollup.rest.RestGetRollupIndexCapsAction;
 import org.elasticsearch.xpack.rollup.rest.RestGetRollupJobsAction;
 import org.elasticsearch.xpack.rollup.rest.RestPutRollupJobAction;
 import org.elasticsearch.xpack.rollup.rest.RestRollupSearchAction;
@@ -136,13 +139,14 @@ public class Rollup extends Plugin implements ActionPlugin, PersistentTaskPlugin
         }
 
         return Arrays.asList(
-                new RestRollupSearchAction(settings, restController),
-                new RestPutRollupJobAction(settings, restController),
-                new RestStartRollupJobAction(settings, restController),
-                new RestStopRollupJobAction(settings, restController),
-                new RestDeleteRollupJobAction(settings, restController),
-                new RestGetRollupJobsAction(settings, restController),
-                new RestGetRollupCapsAction(settings, restController)
+            new RestRollupSearchAction(settings, restController),
+            new RestPutRollupJobAction(settings, restController),
+            new RestStartRollupJobAction(settings, restController),
+            new RestStopRollupJobAction(settings, restController),
+            new RestDeleteRollupJobAction(settings, restController),
+            new RestGetRollupJobsAction(settings, restController),
+            new RestGetRollupCapsAction(settings, restController),
+            new RestGetRollupIndexCapsAction(settings, restController)
         );
 
     }
@@ -153,13 +157,14 @@ public class Rollup extends Plugin implements ActionPlugin, PersistentTaskPlugin
             return emptyList();
         }
         return Arrays.asList(
-                new ActionHandler<>(RollupSearchAction.INSTANCE, TransportRollupSearchAction.class),
-                new ActionHandler<>(PutRollupJobAction.INSTANCE, TransportPutRollupJobAction.class),
-                new ActionHandler<>(StartRollupJobAction.INSTANCE, TransportStartRollupAction.class),
-                new ActionHandler<>(StopRollupJobAction.INSTANCE, TransportStopRollupAction.class),
-                new ActionHandler<>(DeleteRollupJobAction.INSTANCE, TransportDeleteRollupJobAction.class),
-                new ActionHandler<>(GetRollupJobsAction.INSTANCE, TransportGetRollupJobAction.class),
-                new ActionHandler<>(GetRollupCapsAction.INSTANCE, TransportGetRollupCapsAction.class)
+            new ActionHandler<>(RollupSearchAction.INSTANCE, TransportRollupSearchAction.class),
+            new ActionHandler<>(PutRollupJobAction.INSTANCE, TransportPutRollupJobAction.class),
+            new ActionHandler<>(StartRollupJobAction.INSTANCE, TransportStartRollupAction.class),
+            new ActionHandler<>(StopRollupJobAction.INSTANCE, TransportStopRollupAction.class),
+            new ActionHandler<>(DeleteRollupJobAction.INSTANCE, TransportDeleteRollupJobAction.class),
+            new ActionHandler<>(GetRollupJobsAction.INSTANCE, TransportGetRollupJobAction.class),
+            new ActionHandler<>(GetRollupCapsAction.INSTANCE, TransportGetRollupCapsAction.class),
+            new ActionHandler<>(GetRollupIndexCapsAction.INSTANCE, TransportGetRollupIndexCapsAction.class)
         );
     }
 
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java
index 5f013e8897b..6d565e43b86 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java
@@ -44,7 +44,6 @@ public class TransportGetRollupCapsAction extends HandledTransportAction<GetRoll
 
     @Override
     protected void doExecute(Task task, GetRollupCapsAction.Request request, ActionListener<GetRollupCapsAction.Response> listener) {
-
         Map<String, RollableIndexCaps> allCaps = getCaps(request.getIndexPattern(), clusterService.state().getMetaData().indices());
         listener.onResponse(new GetRollupCapsAction.Response(allCaps));
     }
@@ -67,7 +66,7 @@ public class TransportGetRollupCapsAction extends HandledTransportAction<GetRoll
 
                 jobCaps.forEach(jobCap -> {
                     String pattern = indexPattern.equals(MetaData.ALL)
-                            ? jobCap.getIndexPattern() : indexPattern;
+                        ? jobCap.getIndexPattern() : indexPattern;
 
                     // Do we already have an entry for this index pattern?
                     RollableIndexCaps indexCaps = allCaps.get(pattern);
@@ -98,11 +97,11 @@ public class TransportGetRollupCapsAction extends HandledTransportAction<GetRoll
         }
 
         RollupIndexCaps caps = RollupIndexCaps.parseMetadataXContent(
-                new BytesArray(rollupMapping.source().uncompressed()), indexName);
+            new BytesArray(rollupMapping.source().uncompressed()), indexName);
 
         if (caps.hasCaps()) {
             return Optional.of(caps);
         }
         return Optional.empty();
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
new file mode 100644
index 00000000000..5d81f4046eb
--- /dev/null
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.rollup.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
+import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.function.Supplier;
+import java.util.stream.StreamSupport;
+
+public class TransportGetRollupIndexCapsAction extends HandledTransportAction<GetRollupIndexCapsAction.Request,
+    GetRollupIndexCapsAction.Response> {
+
+    private final ClusterService clusterService;
+
+    @Inject
+    public TransportGetRollupIndexCapsAction(Settings settings, TransportService transportService,
+                                             ClusterService clusterService, ActionFilters actionFilters) {
+        super(settings, GetRollupIndexCapsAction.NAME, transportService, actionFilters,
+            (Supplier<GetRollupIndexCapsAction.Request>) GetRollupIndexCapsAction.Request::new);
+        this.clusterService = clusterService;
+    }
+
+    @Override
+    protected void doExecute(Task task, GetRollupIndexCapsAction.Request request,
+                             ActionListener<GetRollupIndexCapsAction.Response> listener) {
+
+        IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(clusterService.getSettings());
+        String[] indices = resolver.concreteIndexNames(clusterService.state(),
+            request.indicesOptions(), request.indices());
+        Map<String, RollableIndexCaps> allCaps = getCapsByRollupIndex(Arrays.asList(indices),
+            clusterService.state().getMetaData().indices());
+        listener.onResponse(new GetRollupIndexCapsAction.Response(allCaps));
+    }
+
+    static Map<String, RollableIndexCaps> getCapsByRollupIndex(List<String> resolvedIndexNames,
+                                                               ImmutableOpenMap<String, IndexMetaData> indices) {
+        Map<String, RollableIndexCaps> allCaps = new TreeMap<>();
+
+        StreamSupport.stream(indices.spliterator(), false)
+            .filter(entry -> resolvedIndexNames.contains(entry.key))
+            .forEach(entry -> {
+                // Does this index have rollup metadata?
+                TransportGetRollupCapsAction.findRollupIndexCaps(entry.key, entry.value)
+                    .ifPresent(cap -> {
+                        cap.getJobCaps().forEach(jobCap -> {
+                            // Do we already have an entry for this index?
+                            RollableIndexCaps indexCaps = allCaps.get(jobCap.getRollupIndex());
+                            if (indexCaps == null) {
+                                indexCaps = new RollableIndexCaps(jobCap.getRollupIndex());
+                            }
+                            indexCaps.addJobCap(jobCap);
+                            allCaps.put(jobCap.getRollupIndex(), indexCaps);
+                        });
+                    });
+            });
+
+        return allCaps;
+    }
+
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java
new file mode 100644
index 00000000000..4f4336f11ab
--- /dev/null
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.rollup.rest;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
+
+public class RestGetRollupIndexCapsAction extends BaseRestHandler {
+    public static final ParseField INDEX = new ParseField("index");
+
+    public RestGetRollupIndexCapsAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.GET, "/{index}/_xpack/rollup/data", this);
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        String index = restRequest.param(INDEX.getPreferredName());
+        IndicesOptions options = IndicesOptions.fromRequest(restRequest, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED);
+        GetRollupIndexCapsAction.Request request = new GetRollupIndexCapsAction.Request(new String[]{index}, options);
+        return channel -> client.execute(GetRollupIndexCapsAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+
+    @Override
+    public String getName() {
+        return "rollup_get_caps_action";
+    }
+}
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
index e63650397ab..e3a45dbd66b 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
@@ -11,11 +11,11 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction;
 import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.mockito.Mockito;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
new file mode 100644
index 00000000000..2066d664996
--- /dev/null
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.rollup.action;
+
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
+import org.elasticsearch.xpack.core.rollup.RollupField;
+import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
+import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.rollup.action.TransportGetRollupIndexCapsAction.getCapsByRollupIndex;
+import static org.hamcrest.Matchers.equalTo;
+
+
+public class GetRollupIndexCapsActionRequestTests extends AbstractStreamableTestCase<GetRollupIndexCapsAction.Request> {
+
+    @Override
+    protected GetRollupIndexCapsAction.Request createTestInstance() {
+        if (randomBoolean()) {
+            return new GetRollupIndexCapsAction.Request(new String[]{MetaData.ALL});
+        }
+        return new GetRollupIndexCapsAction.Request(new String[]{randomAlphaOfLengthBetween(1, 20)});
+    }
+
+    @Override
+    protected GetRollupIndexCapsAction.Request createBlankInstance() {
+        return new GetRollupIndexCapsAction.Request();
+    }
+
+
+    public void testNoIndicesByRollup() {
+        ImmutableOpenMap<String, IndexMetaData> indices = new ImmutableOpenMap.Builder<String, IndexMetaData>().build();
+        Map<String, RollableIndexCaps> caps = getCapsByRollupIndex(Collections.singletonList("foo"), indices);
+        assertThat(caps.size(), equalTo(0));
+    }
+
+    public void testAllIndicesByRollupSingleRollup() throws IOException {
+        int num = randomIntBetween(1,5);
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = new ImmutableOpenMap.Builder<>(5);
+        int indexCounter = 0;
+        for (int j = 0; j < 5; j++) {
+
+            Map<String, Object> jobs = new HashMap<>(num);
+            for (int i = 0; i < num; i++) {
+                String jobName = randomAlphaOfLength(10);
+                String indexName = Integer.toString(indexCounter);
+                indexCounter += 1;
+                jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setRollupIndex("foo").build());
+            }
+
+            MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
+                Collections.singletonMap(RollupField.TYPE_NAME,
+                    Collections.singletonMap("_meta",
+                        Collections.singletonMap(RollupField.ROLLUP_META, jobs))));
+
+            ImmutableOpenMap.Builder<String, MappingMetaData> mappings = ImmutableOpenMap.builder(1);
+            mappings.put(RollupField.TYPE_NAME, mappingMeta);
+            IndexMetaData meta = Mockito.mock(IndexMetaData.class);
+            Mockito.when(meta.getMappings()).thenReturn(mappings.build());
+            indices.put("foo", meta);
+        }
+
+        Map<String, RollableIndexCaps> caps = getCapsByRollupIndex(Collections.singletonList("foo"),
+            indices.build());
+        assertThat(caps.size(), equalTo(1));
+    }
+
+    public void testAllIndicesByRollupManyRollup() throws IOException {
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = new ImmutableOpenMap.Builder<>(5);
+        int indexCounter = 0;
+        for (int j = 0; j < 5; j++) {
+
+            Map<String, Object> jobs = new HashMap<>(1);
+            String jobName = randomAlphaOfLength(10);
+            String indexName = Integer.toString(indexCounter);
+            indexCounter += 1;
+            jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName)
+                .setIndexPattern(indexName)
+                .setRollupIndex("rollup_" + indexName).build());
+
+
+            MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
+                Collections.singletonMap(RollupField.TYPE_NAME,
+                    Collections.singletonMap("_meta",
+                        Collections.singletonMap(RollupField.ROLLUP_META, jobs))));
+
+            ImmutableOpenMap.Builder<String, MappingMetaData> mappings = ImmutableOpenMap.builder(1);
+            mappings.put(RollupField.TYPE_NAME, mappingMeta);
+            IndexMetaData meta = Mockito.mock(IndexMetaData.class);
+            Mockito.when(meta.getMappings()).thenReturn(mappings.build());
+            indices.put("rollup_" + indexName, meta);
+        }
+
+        Map<String, RollableIndexCaps> caps = getCapsByRollupIndex(Arrays.asList(indices.keys().toArray(String.class)), indices.build());
+        assertThat(caps.size(), equalTo(5));
+    }
+
+
+    public void testOneIndexByRollupManyRollup() throws IOException {
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = new ImmutableOpenMap.Builder<>(5);
+        int indexCounter = 0;
+        for (int j = 0; j < 5; j++) {
+
+            Map<String, Object> jobs = new HashMap<>(1);
+            String jobName = randomAlphaOfLength(10);
+            String indexName = Integer.toString(indexCounter);
+            indexCounter += 1;
+            jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName)
+                .setIndexPattern("foo_" + indexName)
+                .setRollupIndex("rollup_" + indexName).build());
+
+            MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
+                Collections.singletonMap(RollupField.TYPE_NAME,
+                    Collections.singletonMap("_meta",
+                        Collections.singletonMap(RollupField.ROLLUP_META, jobs))));
+
+            ImmutableOpenMap.Builder<String, MappingMetaData> mappings = ImmutableOpenMap.builder(1);
+            mappings.put(RollupField.TYPE_NAME, mappingMeta);
+            IndexMetaData meta = Mockito.mock(IndexMetaData.class);
+            Mockito.when(meta.getMappings()).thenReturn(mappings.build());
+            indices.put("rollup_" + indexName, meta);
+        }
+
+        Map<String, RollableIndexCaps> caps = getCapsByRollupIndex(Collections.singletonList("rollup_1"), indices.build());
+        assertThat(caps.size(), equalTo(1));
+        assertThat(caps.get("rollup_1").getIndexName(), equalTo("rollup_1"));
+        assertThat(caps.get("rollup_1").getJobCaps().size(), equalTo(1));
+    }
+
+    public void testOneIndexByRollupOneRollup() throws IOException {
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = new ImmutableOpenMap.Builder<>(5);
+        int indexCounter = 0;
+        for (int j = 0; j < 5; j++) {
+
+            Map<String, Object> jobs = new HashMap<>(1);
+            String jobName = randomAlphaOfLength(10);
+            String indexName = Integer.toString(indexCounter);
+            indexCounter += 1;
+            jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName)
+                .setIndexPattern("foo_" + indexName)
+                .setRollupIndex("rollup_foo").build());
+
+            MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME,
+                Collections.singletonMap(RollupField.TYPE_NAME,
+                    Collections.singletonMap("_meta",
+                        Collections.singletonMap(RollupField.ROLLUP_META, jobs))));
+
+            ImmutableOpenMap.Builder<String, MappingMetaData> mappings = ImmutableOpenMap.builder(1);
+            mappings.put(RollupField.TYPE_NAME, mappingMeta);
+            IndexMetaData meta = Mockito.mock(IndexMetaData.class);
+            Mockito.when(meta.getMappings()).thenReturn(mappings.build());
+            indices.put("rollup_foo", meta);
+        }
+
+        Map<String, RollableIndexCaps> caps = getCapsByRollupIndex(Collections.singletonList("rollup_foo"), indices.build());
+        assertThat(caps.size(), equalTo(1));
+        assertThat(caps.get("rollup_foo").getIndexName(), equalTo("rollup_foo"));
+        assertThat(caps.get("rollup_foo").getJobCaps().size(), equalTo(1));
+    }
+}
+
+
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json
new file mode 100644
index 00000000000..458311417d4
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json
@@ -0,0 +1,17 @@
+{
+  "xpack.rollup.get_rollup_index_caps": {
+    "documentation": "",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/{index}/_xpack/rollup/data",
+      "paths": [ "/{index}/_xpack/rollup/data" ],
+      "parts": {
+        "index": {
+          "type": "string",
+          "required": true,
+          "description": "The rollup index or index pattern to obtain rollup capabilities from."
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml
index 050e49bc4b4..f8bb401a772 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml
@@ -22,6 +22,18 @@ setup:
                 type: date
               value_field:
                 type: integer
+
+  - do:
+      indices.create:
+        index: foo3
+        include_type_name: false
+        body:
+          mappings:
+            properties:
+              the_field:
+                type: date
+              value_field:
+                type: integer
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -228,4 +240,3 @@ setup:
                 - agg: "min"
                 - agg: "max"
                 - agg: "sum"
-
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml
new file mode 100644
index 00000000000..7a539edcc67
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml
@@ -0,0 +1,363 @@
+setup:
+  - do:
+      indices.create:
+        index: foo
+        include_type_name: false
+        body:
+          mappings:
+            properties:
+              the_field:
+                type: date
+              value_field:
+                type: integer
+
+  - do:
+      indices.create:
+        index: foo2
+        include_type_name: false
+        body:
+          mappings:
+            properties:
+              the_field:
+                type: date
+              value_field:
+                type: integer
+
+  - do:
+      indices.create:
+        index: foo3
+        include_type_name: false
+        body:
+          mappings:
+            properties:
+              the_field:
+                type: date
+              value_field:
+                type: integer
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: foo
+        body:  >
+          {
+            "index_pattern": "foo",
+            "rollup_index": "foo_rollup",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "the_field",
+                "interval": "1h"
+              }
+            },
+            "metrics": [
+              {
+                "field": "value_field",
+                "metrics": ["min", "max", "sum"]
+              }
+            ]
+          }
+
+---
+"Verify one job caps by rollup index":
+
+  - do:
+      xpack.rollup.get_rollup_index_caps:
+        index: "foo_rollup"
+
+  - match:
+      foo_rollup:
+        rollup_jobs:
+          - job_id: "foo"
+            rollup_index: "foo_rollup"
+            index_pattern: "foo"
+            fields:
+              the_field:
+                - agg: "date_histogram"
+                  interval: "1h"
+                  time_zone: "UTC"
+              value_field:
+                - agg: "min"
+                - agg: "max"
+                - agg: "sum"
+
+---
+"Verify two job caps by rollup index":
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: foo2
+        body:  >
+          {
+            "index_pattern": "foo",
+            "rollup_index": "foo_rollup",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "the_field",
+                "interval": "1h"
+              }
+            },
+            "metrics": [
+              {
+                "field": "value_field",
+                "metrics": ["min", "max", "sum"]
+              }
+            ]
+          }
+  - do:
+      xpack.rollup.get_rollup_index_caps:
+        index: "foo_rollup"
+
+  - match:
+      foo_rollup:
+        rollup_jobs:
+          - job_id: "foo"
+            rollup_index: "foo_rollup"
+            index_pattern: "foo"
+            fields:
+              the_field:
+                - agg: "date_histogram"
+                  interval: "1h"
+                  time_zone: "UTC"
+              value_field:
+                - agg: "min"
+                - agg: "max"
+                - agg: "sum"
+          - job_id: "foo2"
+            rollup_index: "foo_rollup"
+            index_pattern: "foo"
+            fields:
+              the_field:
+                - agg: "date_histogram"
+                  interval: "1h"
+                  time_zone: "UTC"
+              value_field:
+                - agg: "min"
+                - agg: "max"
+                - agg: "sum"
+
+
+---
+"Verify two different job caps by rollup index":
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: foo2
+        body:  >
+          {
+            "index_pattern": "foo2",
+            "rollup_index": "foo_rollup2",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "the_field",
+                "interval": "1h"
+              }
+            },
+            "metrics": [
+              {
+                "field": "value_field",
+                "metrics": ["min", "max", "sum"]
+              }
+            ]
+          }
+  - do:
+      xpack.rollup.get_rollup_index_caps:
+        index: "foo_rollup"
+
+  - match:
+      foo_rollup:
+        rollup_jobs:
+          - job_id: "foo"
+            rollup_index: "foo_rollup"
+            index_pattern: "foo"
+            fields:
+              the_field:
+                - agg: "date_histogram"
+                  interval: "1h"
+                  time_zone: "UTC"
+              value_field:
+                - agg: "min"
+                - agg: "max"
+                - agg: "sum"
+
+---
+"Verify all job caps by rollup index":
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: foo2
+        body:  >
+          {
+            "index_pattern": "foo2",
+            "rollup_index": "foo_rollup",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "the_field",
+                "interval": "1h"
+              }
+            },
+            "metrics": [
+              {
+                "field": "value_field",
+                "metrics": ["min", "max", "sum"]
+              }
+            ]
+          }
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: foo3
+        body:  >
+          {
+            "index_pattern": "foo3",
+            "rollup_index": "foo_rollup2",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "the_field",
+                "interval": "1h"
+              }
+            },
+            "metrics": [
+              {
+                "field": "value_field",
+                "metrics": ["min", "max", "sum"]
+              }
+            ]
+          }
+
+  - do:
+      xpack.rollup.get_rollup_index_caps:
+        index: "_all"
+
+  - match:
+      $body:
+        foo_rollup:
+          rollup_jobs:
+            - job_id: "foo"
+              rollup_index: "foo_rollup"
+              index_pattern: "foo"
+              fields:
+                the_field:
+                  - agg: "date_histogram"
+                    interval: "1h"
+                    time_zone: "UTC"
+                value_field:
+                  - agg: "min"
+                  - agg: "max"
+                  - agg: "sum"
+            - job_id: "foo2"
+              rollup_index: "foo_rollup"
+              index_pattern: "foo2"
+              fields:
+                the_field:
+                  - agg: "date_histogram"
+                    interval: "1h"
+                    time_zone: "UTC"
+                value_field:
+                  - agg: "min"
+                  - agg: "max"
+                  - agg: "sum"
+        foo_rollup2:
+          rollup_jobs:
+            - job_id: "foo3"
+              rollup_index: "foo_rollup2"
+              index_pattern: "foo3"
+              fields:
+                the_field:
+                  - agg: "date_histogram"
+                    interval: "1h"
+                    time_zone: "UTC"
+                value_field:
+                  - agg: "min"
+                  - agg: "max"
+                  - agg: "sum"
+
+---
+"Verify index pattern":
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: foo2
+        body:  >
+          {
+            "index_pattern": "foo2",
+            "rollup_index": "foo_rollup",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "the_field",
+                "interval": "1h"
+              }
+            },
+            "metrics": [
+              {
+                "field": "value_field",
+                "metrics": ["min", "max", "sum"]
+              }
+            ]
+          }
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: foo3
+        body:  >
+          {
+            "index_pattern": "foo3",
+            "rollup_index": "foo_rollup2",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "the_field",
+                "interval": "1h"
+              }
+            },
+            "metrics": [
+              {
+                "field": "value_field",
+                "metrics": ["min", "max", "sum"]
+              }
+            ]
+          }
+
+  - do:
+      xpack.rollup.get_rollup_index_caps:
+        index: "*_rollup2"
+
+  - match:
+      $body:
+        foo_rollup2:
+          rollup_jobs:
+            - job_id: "foo3"
+              rollup_index: "foo_rollup2"
+              index_pattern: "foo3"
+              fields:
+                the_field:
+                  - agg: "date_histogram"
+                    interval: "1h"
+                    time_zone: "UTC"
+                value_field:
+                  - agg: "min"
+                  - agg: "max"
+                  - agg: "sum"
+

From d596447f3dc186855af1e32dc06a75d06feea89c Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 16 Jul 2018 17:44:19 -0400
Subject: [PATCH 048/260] Switch non-x-pack to new style requests (#32106)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes most of the calls not in X-Pack to their new versions.
---
 .../rest/Netty4BadRequestIT.java              | 19 +++++++-----
 .../rest/Netty4HeadBodyIsEmptyIT.java         | 30 ++++++++++++-------
 .../hdfs/HaHdfsFailoverTestSuiteIT.java       | 20 +++++--------
 .../VerifyVersionConstantsIT.java             |  3 +-
 4 files changed, 39 insertions(+), 33 deletions(-)

diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
index bc89558d3c6..17a62b3a440 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java
@@ -19,7 +19,8 @@
 
 package org.elasticsearch.rest;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Setting;
@@ -43,7 +44,7 @@ import static org.hamcrest.Matchers.hasToString;
 public class Netty4BadRequestIT extends ESRestTestCase {
 
     public void testBadRequest() throws IOException {
-        final Response response = client().performRequest("GET", "/_nodes/settings", Collections.emptyMap());
+        final Response response = client().performRequest(new Request("GET", "/_nodes/settings"));
         final ObjectPath objectPath = ObjectPath.createFromResponse(response);
         final Map<String, Object> map = objectPath.evaluate("nodes");
         int maxMaxInitialLineLength = Integer.MIN_VALUE;
@@ -77,9 +78,9 @@ public class Netty4BadRequestIT extends ESRestTestCase {
     }
 
     public void testInvalidParameterValue() throws IOException {
-        final ResponseException e = expectThrows(
-                ResponseException.class,
-                () -> client().performRequest("GET", "/_cluster/settings", Collections.singletonMap("pretty", "neither-true-nor-false")));
+        final Request request = new Request("GET", "/_cluster/settings");
+        request.addParameter("pretty", "neither-true-nor-false");
+        final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
         final Response response = e.getResponse();
         assertThat(response.getStatusLine().getStatusCode(), equalTo(400));
         final ObjectPath objectPath = ObjectPath.createFromResponse(response);
@@ -89,9 +90,11 @@ public class Netty4BadRequestIT extends ESRestTestCase {
     }
 
     public void testInvalidHeaderValue() throws IOException {
-        final BasicHeader header = new BasicHeader("Content-Type", "\t");
-        final ResponseException e =
-                expectThrows(ResponseException.class, () -> client().performRequest("GET", "/_cluster/settings", header));
+        final Request request = new Request("GET", "/_cluster/settings");
+        final RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Content-Type", "\t");
+        request.setOptions(options);
+        final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
         final Response response = e.getResponse();
         assertThat(response.getStatusLine().getStatusCode(), equalTo(400));
         final ObjectPath objectPath = ObjectPath.createFromResponse(response);
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
index 20b18ebdadd..17b374ecb37 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
@@ -19,8 +19,7 @@
 
 package org.elasticsearch.rest;
 
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -57,8 +56,9 @@ public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase {
                 builder.field("test", "test");
             }
             builder.endObject();
-            client().performRequest("PUT", "/" + indexName + "/" + typeName + "/" + "1", emptyMap(),
-                new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON));
+            Request request = new Request("PUT", "/" + indexName + "/" + typeName + "/" + "1");
+            request.setJsonEntity(Strings.toString(builder));
+            client().performRequest(request);
         }
     }
 
@@ -109,8 +109,9 @@ public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase {
             }
             builder.endObject();
 
-            client().performRequest("POST", "_aliases", emptyMap(), new StringEntity(Strings.toString(builder),
-                            ContentType.APPLICATION_JSON));
+            Request request = new Request("POST", "/_aliases");
+            request.setJsonEntity(Strings.toString(builder));
+            client().performRequest(request);
             headTestCase("/_alias/test_alias", emptyMap(), greaterThan(0));
             headTestCase("/test/_alias/test_alias", emptyMap(), greaterThan(0));
         }
@@ -135,8 +136,9 @@ public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase {
             }
             builder.endObject();
 
-            client().performRequest("PUT", "/_template/template", emptyMap(),
-                new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON));
+            Request request = new Request("PUT", "/_template/template");
+            request.setJsonEntity(Strings.toString(builder));
+            client().performRequest(request);
             headTestCase("/_template/template", emptyMap(), greaterThan(0));
         }
     }
@@ -164,8 +166,10 @@ public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase {
                 builder.endObject();
             }
             builder.endObject();
-            client().performRequest("PUT", "/test-no-source", emptyMap(), new StringEntity(Strings.toString(builder),
-                            ContentType.APPLICATION_JSON));
+
+            Request request = new Request("PUT", "/test-no-source");
+            request.setJsonEntity(Strings.toString(builder));
+            client().performRequest(request);
             createTestDoc("test-no-source", "test-no-source");
             headTestCase("/test-no-source/test-no-source/1/_source", emptyMap(), NOT_FOUND.getStatus(), equalTo(0));
         }
@@ -190,7 +194,11 @@ public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase {
             final Map<String, String> params,
             final int expectedStatusCode,
             final Matcher<Integer> matcher) throws IOException {
-        Response response = client().performRequest("HEAD", url, params);
+        Request request = new Request("HEAD", url);
+        for (Map.Entry<String, String> param : params.entrySet()) {
+            request.addParameter(param.getKey(), param.getValue());
+        }
+        Response response = client().performRequest(request);
         assertEquals(expectedStatusCode, response.getStatusLine().getStatusCode());
         assertThat(Integer.valueOf(response.getHeader("Content-Length")), matcher);
         assertNull("HEAD requests shouldn't have a response body but " + url + " did", response.getEntity());
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java
index ce4fe9b6d3f..0248576b573 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java
@@ -21,16 +21,13 @@ package org.elasticsearch.repositories.hdfs;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ha.BadFencingConfigurationException;
@@ -42,9 +39,7 @@ import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdfs.tools.DFSHAAdmin;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.http.Header;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.io.PathUtils;
@@ -58,8 +53,6 @@ public class HaHdfsFailoverTestSuiteIT extends ESRestTestCase {
 
     public void testHAFailoverWithRepository() throws Exception {
         RestClient client = client();
-        Map<String, String> emptyParams = Collections.emptyMap();
-        Header contentHeader = new BasicHeader("Content-Type", "application/json");
 
         String esKerberosPrincipal = System.getProperty("test.krb5.principal.es");
         String hdfsKerberosPrincipal = System.getProperty("test.krb5.principal.hdfs");
@@ -106,7 +99,8 @@ public class HaHdfsFailoverTestSuiteIT extends ESRestTestCase {
 
         // Create repository
         {
-            Response response = client.performRequest("PUT", "/_snapshot/hdfs_ha_repo_read", emptyParams, new NStringEntity(
+            Request request = new Request("PUT", "/_snapshot/hdfs_ha_repo_read");
+            request.setJsonEntity(
                 "{" +
                     "\"type\":\"hdfs\"," +
                     "\"settings\":{" +
@@ -121,15 +115,15 @@ public class HaHdfsFailoverTestSuiteIT extends ESRestTestCase {
                         "\"conf.dfs.client.failover.proxy.provider.ha-hdfs\": " +
                             "\"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"" +
                     "}" +
-                "}",
-                Charset.defaultCharset()), contentHeader);
+                "}");
+            Response response = client.performRequest(request);
 
             Assert.assertEquals(200, response.getStatusLine().getStatusCode());
         }
 
         // Get repository
         {
-            Response response = client.performRequest("GET", "/_snapshot/hdfs_ha_repo_read/_all", emptyParams);
+            Response response = client.performRequest(new Request("GET", "/_snapshot/hdfs_ha_repo_read/_all"));
             Assert.assertEquals(200, response.getStatusLine().getStatusCode());
         }
 
@@ -138,7 +132,7 @@ public class HaHdfsFailoverTestSuiteIT extends ESRestTestCase {
 
         // Get repository again
         {
-            Response response = client.performRequest("GET", "/_snapshot/hdfs_ha_repo_read/_all", emptyParams);
+            Response response = client.performRequest(new Request("GET", "/_snapshot/hdfs_ha_repo_read/_all"));
             Assert.assertEquals(200, response.getStatusLine().getStatusCode());
         }
     }
diff --git a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java
index a26237962bf..71fc9d09fe3 100644
--- a/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java
+++ b/qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.qa.verify_version_constants;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
@@ -32,7 +33,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 public class VerifyVersionConstantsIT extends ESRestTestCase {
 
     public void testLuceneVersionConstant() throws IOException, ParseException {
-        final Response response = client().performRequest("GET", "/");
+        final Response response = client().performRequest(new Request("GET", "/"));
         assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
         final ObjectPath objectPath = ObjectPath.createFromResponse(response);
         final String elasticsearchVersionString = objectPath.evaluate("version.number").toString();

From f699cb9f5565bfe6825ae196969a45b4e6365d31 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 17 Jul 2018 00:26:01 +0200
Subject: [PATCH 049/260] Bypass highlight query terms extraction on empty
 fields (#32090)

Dealing with empty fields in the highlight phase can
slow down the query because the query terms extraction is done independently
on each field. This change shortcuts the highlighting performed by the unified highlighter
for fields that are not present in the document. In such cases there is nothing to higlight so
we don't need to visit the query to build the highligh builder.
---
 .../search/fetch/subphase/highlight/UnifiedHighlighter.java    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
index 7f209ed0586..2c9d482cab0 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -76,6 +76,9 @@ public class UnifiedHighlighter implements Highlighter {
             fieldValues = fieldValues.stream()
                 .map((s) -> convertFieldValue(fieldType, s))
                 .collect(Collectors.toList());
+            if (fieldValues.size() == 0) {
+                return null;
+            }
             final IndexSearcher searcher = new IndexSearcher(hitContext.reader());
             final CustomUnifiedHighlighter highlighter;
             final String fieldValue = mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);

From 15740d622971551e6595017be21dc6dc58a6ea84 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 16 Jul 2018 16:13:48 -0700
Subject: [PATCH 050/260] Painless: Move and Rename Several Methods in the
 lookup package (#32105)

---
 .../painless/AnalyzerCaster.java              |   7 +-
 .../java/org/elasticsearch/painless/Def.java  |   9 +-
 .../elasticsearch/painless/FunctionRef.java   |   3 +-
 .../org/elasticsearch/painless/Locals.java    |   5 +-
 .../painless/ScriptClassInfo.java             |   3 +-
 .../painless/antlr/EnhancedPainlessLexer.java |   2 +-
 .../painless/lookup/PainlessLookup.java       | 208 +------------
 .../lookup/PainlessLookupBuilder.java         |  69 +----
 .../lookup/PainlessLookupUtility.java         | 284 ++++++++++++++++++
 .../painless/lookup/PainlessMethod.java       |  14 +-
 .../painless/node/AExpression.java            |   6 +-
 .../painless/node/EAssignment.java            |   4 +-
 .../elasticsearch/painless/node/EBinary.java  |  37 ++-
 .../painless/node/ECallLocal.java             |   4 +-
 .../painless/node/ECapturingFunctionRef.java  |  11 +-
 .../elasticsearch/painless/node/ECast.java    |   6 +-
 .../elasticsearch/painless/node/EComp.java    |  28 +-
 .../painless/node/EFunctionRef.java           |  10 +-
 .../painless/node/EInstanceof.java            |   7 +-
 .../elasticsearch/painless/node/ELambda.java  |  20 +-
 .../painless/node/EListInit.java              |   6 +-
 .../elasticsearch/painless/node/EMapInit.java |   6 +-
 .../elasticsearch/painless/node/ENewObj.java  |   4 +-
 .../elasticsearch/painless/node/ENull.java    |   4 +-
 .../elasticsearch/painless/node/EUnary.java   |  15 +-
 .../elasticsearch/painless/node/PBrace.java   |   8 +-
 .../painless/node/PCallInvoke.java            |  12 +-
 .../elasticsearch/painless/node/PField.java   |  16 +-
 .../painless/node/PSubCallInvoke.java         |   2 +-
 .../painless/node/PSubDefArray.java           |   2 +-
 .../painless/node/PSubDefCall.java            |   2 +-
 .../painless/node/PSubDefField.java           |   2 +-
 .../painless/node/PSubField.java              |   8 +-
 .../painless/node/PSubListShortcut.java       |   4 +-
 .../painless/node/PSubMapShortcut.java        |   4 +-
 .../painless/node/PSubShortcut.java           |   2 +-
 .../elasticsearch/painless/node/SEach.java    |   8 +-
 .../painless/node/SFunction.java              |  11 +-
 .../painless/node/SSubEachArray.java          |   6 +-
 .../painless/node/SSubEachIterable.java       |  16 +-
 40 files changed, 468 insertions(+), 407 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 457ec82a5e4..6cfc7ff6ebf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.def;
 
 import java.util.Objects;
@@ -465,8 +465,9 @@ public final class AnalyzerCaster {
                 (actual.isAssignableFrom(expected) && explicit)) {
             return PainlessCast.standard(actual, expected, explicit);
         } else {
-            throw location.createError(new ClassCastException(
-                    "Cannot cast from [" + PainlessLookup.ClassToName(actual) + "] to [" + PainlessLookup.ClassToName(expected) + "]."));
+            throw location.createError(new ClassCastException("Cannot cast from " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(actual) + "] to " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "]."));
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index f006b57f46b..78db712d183 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.lang.invoke.CallSite;
@@ -302,7 +303,7 @@ public final class Def {
                                                               nestedType,
                                                               0,
                                                               DefBootstrap.REFERENCE,
-                                                              PainlessLookup.ClassToName(interfaceType));
+                                                              PainlessLookupUtility.anyTypeToPainlessTypeName(interfaceType));
                      filter = nested.dynamicInvoker();
                  } else {
                      throw new AssertionError();
@@ -347,7 +348,7 @@ public final class Def {
              PainlessMethod interfaceMethod = painlessLookup.getPainlessStructFromJavaClass(clazz).functionalMethod;
              if (interfaceMethod == null) {
                  throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                                                    "to [" + PainlessLookup.ClassToName(clazz) + "], not a functional interface");
+                         "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(clazz) + "], not a functional interface");
              }
              int arity = interfaceMethod.arguments.size() + captures.length;
              final MethodHandle handle;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 6ab7292c7f6..9e72dc2c835 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.objectweb.asm.Type;
@@ -168,7 +169,7 @@ public class FunctionRef {
         PainlessMethod method = painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod;
         if (method == null) {
             throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                                               "to [" + PainlessLookup.ClassToName(expected) + "], not a functional interface");
+                    "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], not a functional interface");
         }
 
         // lookup requested method
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index bab1b7942d6..e797740fed1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.painless.ScriptClassInfo.MethodArgument;
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
-import org.elasticsearch.painless.ScriptClassInfo.MethodArgument;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -292,7 +293,7 @@ public final class Locals {
         @Override
         public String toString() {
             StringBuilder b = new StringBuilder();
-            b.append("Variable[type=").append(PainlessLookup.ClassToName(clazz));
+            b.append("Variable[type=").append(PainlessLookupUtility.anyTypeToPainlessTypeName(clazz));
             b.append(",name=").append(name);
             b.append(",slot=").append(slot);
             if (readonly) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
index df43dba7b34..ff2061a9a4b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 
 import java.lang.invoke.MethodType;
 import java.lang.reflect.Field;
@@ -182,7 +183,7 @@ public class ScriptClassInfo {
 
     private static Class<?> definitionTypeForClass(PainlessLookup painlessLookup, Class<?> type,
                                                    Function<Class<?>, String> unknownErrorMessageSource) {
-        type = PainlessLookup.ObjectClassTodefClass(type);
+        type = PainlessLookupUtility.javaObjectTypeToPainlessDefType(type);
         Class<?> componentType = type;
 
         while (componentType.isArray()) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
index e6c5da05798..f1db35636b4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
@@ -23,8 +23,8 @@ import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.LexerNoViableAltException;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.misc.Interval;
-import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.lookup.PainlessLookup;
 
 /**
  * A lexer that is customized for painless. It:
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
index feeaf4d34bc..6111d12317b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.lookup;
 
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
@@ -30,154 +29,6 @@ import java.util.Map;
  */
 public final class PainlessLookup {
 
-    public static Class<?> getBoxedType(Class<?> clazz) {
-        if (clazz == boolean.class) {
-            return Boolean.class;
-        } else if (clazz == byte.class) {
-            return Byte.class;
-        } else if (clazz == short.class) {
-            return Short.class;
-        } else if (clazz == char.class) {
-            return Character.class;
-        } else if (clazz == int.class) {
-            return Integer.class;
-        } else if (clazz == long.class) {
-            return Long.class;
-        } else if (clazz == float.class) {
-            return Float.class;
-        } else if (clazz == double.class) {
-            return Double.class;
-        }
-
-        return clazz;
-    }
-
-    public static Class<?> getUnboxedype(Class<?> clazz) {
-        if (clazz == Boolean.class) {
-            return boolean.class;
-        } else if (clazz == Byte.class) {
-            return byte.class;
-        } else if (clazz == Short.class) {
-            return short.class;
-        } else if (clazz == Character.class) {
-            return char.class;
-        } else if (clazz == Integer.class) {
-            return int.class;
-        } else if (clazz == Long.class) {
-            return long.class;
-        } else if (clazz == Float.class) {
-            return float.class;
-        } else if (clazz == Double.class) {
-            return double.class;
-        }
-
-        return clazz;
-    }
-
-    public static boolean isConstantType(Class<?> clazz) {
-        return clazz == boolean.class ||
-               clazz == byte.class    ||
-               clazz == short.class   ||
-               clazz == char.class    ||
-               clazz == int.class     ||
-               clazz == long.class    ||
-               clazz == float.class   ||
-               clazz == double.class  ||
-               clazz == String.class;
-    }
-
-    public Class<?> getClassFromBinaryName(String painlessType) {
-        return painlessTypesToJavaClasses.get(painlessType.replace('$', '.'));
-    }
-
-    public static Class<?> ObjectClassTodefClass(Class<?> clazz) {
-        if (clazz.isArray()) {
-            Class<?> component = clazz.getComponentType();
-            int dimensions = 1;
-
-            while (component.isArray()) {
-                component = component.getComponentType();
-                ++dimensions;
-            }
-
-            if (component == Object.class) {
-                char[] braces = new char[dimensions];
-                Arrays.fill(braces, '[');
-
-                String descriptor = new String(braces) + org.objectweb.asm.Type.getType(def.class).getDescriptor();
-                org.objectweb.asm.Type type = org.objectweb.asm.Type.getType(descriptor);
-
-                try {
-                    return Class.forName(type.getInternalName().replace('/', '.'));
-                } catch (ClassNotFoundException exception) {
-                    throw new IllegalStateException("internal error", exception);
-                }
-            }
-        } else if (clazz == Object.class) {
-            return def.class;
-        }
-
-        return clazz;
-    }
-
-    public static Class<?> defClassToObjectClass(Class<?> clazz) {
-        if (clazz.isArray()) {
-            Class<?> component = clazz.getComponentType();
-            int dimensions = 1;
-
-            while (component.isArray()) {
-                component = component.getComponentType();
-                ++dimensions;
-            }
-
-            if (component == def.class) {
-                char[] braces = new char[dimensions];
-                Arrays.fill(braces, '[');
-
-                String descriptor = new String(braces) + org.objectweb.asm.Type.getType(Object.class).getDescriptor();
-                org.objectweb.asm.Type type = org.objectweb.asm.Type.getType(descriptor);
-
-                try {
-                    return Class.forName(type.getInternalName().replace('/', '.'));
-                } catch (ClassNotFoundException exception) {
-                    throw new IllegalStateException("internal error", exception);
-                }
-            }
-        } else if (clazz == def.class) {
-            return Object.class;
-        }
-
-        return clazz;
-    }
-
-    public static String ClassToName(Class<?> clazz) {
-        if (clazz.isLocalClass() || clazz.isAnonymousClass()) {
-            return null;
-        } else if (clazz.isArray()) {
-            Class<?> component = clazz.getComponentType();
-            int dimensions = 1;
-
-            while (component.isArray()) {
-                component = component.getComponentType();
-                ++dimensions;
-            }
-
-            if (component == def.class) {
-                StringBuilder builder = new StringBuilder(def.class.getSimpleName());
-
-                for (int dimension = 0; dimension < dimensions; dimension++) {
-                    builder.append("[]");
-                }
-
-                return builder.toString();
-            }
-        } else if (clazz == def.class) {
-            return def.class.getSimpleName();
-        }
-
-        return clazz.getCanonicalName().replace('$', '.');
-    }
-
     public Collection<PainlessClass> getStructs() {
         return javaClassesToPainlessStructs.values();
     }
@@ -190,6 +41,10 @@ public final class PainlessLookup {
         this.javaClassesToPainlessStructs = Collections.unmodifiableMap(javaClassesToPainlessStructs);
     }
 
+    public Class<?> getClassFromBinaryName(String painlessType) {
+        return painlessTypesToJavaClasses.get(painlessType.replace('$', '.'));
+    }
+
     public boolean isSimplePainlessType(String painlessType) {
         return painlessTypesToJavaClasses.containsKey(painlessType);
     }
@@ -199,59 +54,6 @@ public final class PainlessLookup {
     }
 
     public Class<?> getJavaClassFromPainlessType(String painlessType) {
-        Class<?> javaClass = painlessTypesToJavaClasses.get(painlessType);
-
-        if (javaClass != null) {
-            return javaClass;
-        }
-        int arrayDimensions = 0;
-        int arrayIndex = painlessType.indexOf('[');
-
-        if (arrayIndex != -1) {
-            int length = painlessType.length();
-
-            while (arrayIndex < length) {
-                if (painlessType.charAt(arrayIndex) == '[' && ++arrayIndex < length && painlessType.charAt(arrayIndex++) == ']') {
-                    ++arrayDimensions;
-                } else {
-                    throw new IllegalArgumentException("invalid painless type [" + painlessType + "].");
-                }
-            }
-
-            painlessType = painlessType.substring(0, painlessType.indexOf('['));
-            javaClass = painlessTypesToJavaClasses.get(painlessType);
-
-            char braces[] = new char[arrayDimensions];
-            Arrays.fill(braces, '[');
-            String descriptor = new String(braces);
-
-            if (javaClass == boolean.class) {
-                descriptor += "Z";
-            } else if (javaClass == byte.class) {
-                descriptor += "B";
-            } else if (javaClass == short.class) {
-                descriptor += "S";
-            } else if (javaClass == char.class) {
-                descriptor += "C";
-            } else if (javaClass == int.class) {
-                descriptor += "I";
-            } else if (javaClass == long.class) {
-                descriptor += "J";
-            } else if (javaClass == float.class) {
-                descriptor += "F";
-            } else if (javaClass == double.class) {
-                descriptor += "D";
-            } else {
-                descriptor += "L" + javaClass.getName() + ";";
-            }
-
-            try {
-                return Class.forName(descriptor);
-            } catch (ClassNotFoundException cnfe) {
-                throw new IllegalStateException("invalid painless type [" + painlessType + "]", cnfe);
-            }
-        }
-
-        throw new IllegalArgumentException("invalid painless type [" + painlessType + "]");
+        return PainlessLookupUtility.painlessTypeNameToPainlessType(painlessType, painlessTypesToJavaClasses);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 1dadce318d6..9a5e08d65a7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -30,7 +30,6 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -279,7 +278,7 @@ public class PainlessLookupBuilder {
                 Class<?> painlessParameterClass = getJavaClassFromPainlessType(painlessParameterTypeName);
 
                 painlessParametersTypes.add(painlessParameterClass);
-                javaClassParameters[parameterCount] = PainlessLookup.defClassToObjectClass(painlessParameterClass);
+                javaClassParameters[parameterCount] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(painlessParameterClass);
             } catch (IllegalArgumentException iae) {
                 throw new IllegalArgumentException("struct not defined for constructor parameter [" + painlessParameterTypeName + "] " +
                     "with owner struct [" + ownerStructName + "] and constructor parameters " +
@@ -364,7 +363,8 @@ public class PainlessLookupBuilder {
                 Class<?> painlessParameterClass = getJavaClassFromPainlessType(painlessParameterTypeName);
 
                 painlessParametersTypes.add(painlessParameterClass);
-                javaClassParameters[parameterCount + augmentedOffset] = PainlessLookup.defClassToObjectClass(painlessParameterClass);
+                javaClassParameters[parameterCount + augmentedOffset] =
+                        PainlessLookupUtility.painlessDefTypeToJavaObjectType(painlessParameterClass);
             } catch (IllegalArgumentException iae) {
                 throw new IllegalArgumentException("struct not defined for method parameter [" + painlessParameterTypeName + "] " +
                     "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
@@ -393,7 +393,7 @@ public class PainlessLookupBuilder {
                 "and parameters " + whitelistMethod.painlessParameterTypeNames, iae);
         }
 
-        if (javaMethod.getReturnType() != PainlessLookup.defClassToObjectClass(painlessReturnClass)) {
+        if (javaMethod.getReturnType() != PainlessLookupUtility.painlessDefTypeToJavaObjectType(painlessReturnClass)) {
             throw new IllegalArgumentException("specified return type class [" + painlessReturnClass + "] " +
                 "does not match the return type class [" + javaMethod.getReturnType() + "] for the " +
                 "method with name [" + whitelistMethod.javaMethodName + "] " +
@@ -711,64 +711,11 @@ public class PainlessLookupBuilder {
         return painless;
     }
 
-    public Class<?> getJavaClassFromPainlessType(String painlessType) {
-        Class<?> javaClass = painlessTypesToJavaClasses.get(painlessType);
-
-        if (javaClass != null) {
-            return javaClass;
-        }
-        int arrayDimensions = 0;
-        int arrayIndex = painlessType.indexOf('[');
-
-        if (arrayIndex != -1) {
-            int length = painlessType.length();
-
-            while (arrayIndex < length) {
-                if (painlessType.charAt(arrayIndex) == '[' && ++arrayIndex < length && painlessType.charAt(arrayIndex++) == ']') {
-                    ++arrayDimensions;
-                } else {
-                    throw new IllegalArgumentException("invalid painless type [" + painlessType + "].");
-                }
-            }
-
-            painlessType = painlessType.substring(0, painlessType.indexOf('['));
-            javaClass = painlessTypesToJavaClasses.get(painlessType);
-
-            char braces[] = new char[arrayDimensions];
-            Arrays.fill(braces, '[');
-            String descriptor = new String(braces);
-
-            if (javaClass == boolean.class) {
-                descriptor += "Z";
-            } else if (javaClass == byte.class) {
-                descriptor += "B";
-            } else if (javaClass == short.class) {
-                descriptor += "S";
-            } else if (javaClass == char.class) {
-                descriptor += "C";
-            } else if (javaClass == int.class) {
-                descriptor += "I";
-            } else if (javaClass == long.class) {
-                descriptor += "J";
-            } else if (javaClass == float.class) {
-                descriptor += "F";
-            } else if (javaClass == double.class) {
-                descriptor += "D";
-            } else {
-                descriptor += "L" + javaClass.getName() + ";";
-            }
-
-            try {
-                return Class.forName(descriptor);
-            } catch (ClassNotFoundException cnfe) {
-                throw new IllegalStateException("invalid painless type [" + painlessType + "]", cnfe);
-            }
-        }
-
-        throw new IllegalArgumentException("invalid painless type [" + painlessType + "]");
-    }
-
     public PainlessLookup build() {
         return new PainlessLookup(painlessTypesToJavaClasses, javaClassesToPainlessStructs);
     }
+
+    public Class<?> getJavaClassFromPainlessType(String painlessType) {
+        return PainlessLookupUtility.painlessTypeNameToPainlessType(painlessType, painlessTypesToJavaClasses);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
new file mode 100644
index 00000000000..d1f3ee4ece3
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.lookup;
+
+import org.objectweb.asm.Type;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * This class contains methods shared by {@link PainlessLookupBuilder}, {@link PainlessLookup}, and other classes within
+ * Painless for conversion between type names and types along with some other various utility methods.
+ *
+ * The following terminology is used for variable names throughout the lookup package:
+ *
+ * - javaClass           (Class)         - a java class including def and excluding array type java classes
+ * - javaClassName       (String)        - the fully qualified java class name for a javaClass
+ * - painlessClassName   (String)        - the fully qualified painless name or imported painless name for a painlessClass
+ * - anyClassName        (String)        - either a javaClassName or a painlessClassName
+ * - javaType            (Class)         - a java class excluding def and array type java classes
+ * - painlessType        (Class)         - a java class including def and array type java classes
+ * - javaTypeName        (String)        - the fully qualified java Type name for a javaType
+ * - painlessTypeName    (String)        - the fully qualified painless name or imported painless name for a painlessType
+ * - anyTypeName         (String)        - either a javaTypeName or a painlessTypeName
+ * - painlessClass       (PainlessClass) - a painless class object
+ *
+ * Under ambiguous circumstances most variable names are prefixed with asm, java, or painless.
+ * If the variable name is the same for asm, java, and painless, no prefix is used.
+ */
+public final class PainlessLookupUtility {
+
+    public static Class<?> javaObjectTypeToPainlessDefType(Class<?> javaType) {
+        if (javaType.isArray()) {
+            Class<?> javaTypeComponent = javaType.getComponentType();
+            int arrayDimensions = 1;
+
+            while (javaTypeComponent.isArray()) {
+                javaTypeComponent = javaTypeComponent.getComponentType();
+                ++arrayDimensions;
+            }
+
+            if (javaTypeComponent == Object.class) {
+                char[] asmDescriptorBraces = new char[arrayDimensions];
+                Arrays.fill(asmDescriptorBraces, '[');
+
+                String asmDescriptor = new String(asmDescriptorBraces) + Type.getType(def.class).getDescriptor();
+                Type asmType = Type.getType(asmDescriptor);
+
+                try {
+                    return Class.forName(asmType.getInternalName().replace('/', '.'));
+                } catch (ClassNotFoundException cnfe) {
+                    throw new IllegalStateException("internal error", cnfe);
+                }
+            }
+        } else if (javaType == Object.class) {
+            return def.class;
+        }
+
+        return javaType;
+    }
+
+    public static Class<?> painlessDefTypeToJavaObjectType(Class<?> painlessType) {
+        if (painlessType.isArray()) {
+            Class<?> painlessTypeComponent = painlessType.getComponentType();
+            int arrayDimensions = 1;
+
+            while (painlessTypeComponent.isArray()) {
+                painlessTypeComponent = painlessTypeComponent.getComponentType();
+                ++arrayDimensions;
+            }
+
+            if (painlessTypeComponent == def.class) {
+                char[] asmDescriptorBraces = new char[arrayDimensions];
+                Arrays.fill(asmDescriptorBraces, '[');
+
+                String asmDescriptor = new String(asmDescriptorBraces) + Type.getType(Object.class).getDescriptor();
+                Type asmType = Type.getType(asmDescriptor);
+
+                try {
+                    return Class.forName(asmType.getInternalName().replace('/', '.'));
+                } catch (ClassNotFoundException exception) {
+                    throw new IllegalStateException("internal error", exception);
+                }
+            }
+        } else if (painlessType == def.class) {
+            return Object.class;
+        }
+
+        return painlessType;
+    }
+
+    public static String anyTypeNameToPainlessTypeName(String anyTypeName) {
+        return anyTypeName.replace(def.class.getName(), DEF_PAINLESS_CLASS_NAME).replace('$', '.');
+    }
+
+    public static String anyTypeToPainlessTypeName(Class<?> anyType) {
+        if (anyType.isLocalClass() || anyType.isAnonymousClass()) {
+            return null;
+        } else if (anyType.isArray()) {
+            Class<?> anyTypeComponent = anyType.getComponentType();
+            int arrayDimensions = 1;
+
+            while (anyTypeComponent.isArray()) {
+                anyTypeComponent = anyTypeComponent.getComponentType();
+                ++arrayDimensions;
+            }
+
+            if (anyTypeComponent == def.class) {
+                StringBuilder painlessDefTypeNameArrayBuilder = new StringBuilder(DEF_PAINLESS_CLASS_NAME);
+
+                for (int dimension = 0; dimension < arrayDimensions; dimension++) {
+                    painlessDefTypeNameArrayBuilder.append("[]");
+                }
+
+                return painlessDefTypeNameArrayBuilder.toString();
+            }
+        } else if (anyType == def.class) {
+            return DEF_PAINLESS_CLASS_NAME;
+        }
+
+        return anyType.getCanonicalName().replace('$', '.');
+    }
+
+    public static Class<?> painlessTypeNameToPainlessType(String painlessTypeName, Map<String, Class<?>> painlessClassNamesToJavaClasses) {
+        Class<?> javaClass = painlessClassNamesToJavaClasses.get(painlessTypeName);
+
+        if (javaClass != null) {
+            return javaClass;
+        }
+
+        int arrayDimensions = 0;
+        int arrayIndex = painlessTypeName.indexOf('[');
+
+        if (arrayIndex != -1) {
+            int painlessTypeNameLength = painlessTypeName.length();
+
+            while (arrayIndex < painlessTypeNameLength) {
+                if (painlessTypeName.charAt(arrayIndex) == '[' &&
+                    ++arrayIndex < painlessTypeNameLength  &&
+                    painlessTypeName.charAt(arrayIndex++) == ']') {
+                    ++arrayDimensions;
+                } else {
+                    throw new IllegalArgumentException("invalid painless type [" + painlessTypeName + "].");
+                }
+            }
+
+            painlessTypeName = painlessTypeName.substring(0, painlessTypeName.indexOf('['));
+            javaClass = painlessClassNamesToJavaClasses.get(painlessTypeName);
+
+            char javaDescriptorBraces[] = new char[arrayDimensions];
+            Arrays.fill(javaDescriptorBraces, '[');
+            String javaDescriptor = new String(javaDescriptorBraces);
+
+            if (javaClass == boolean.class) {
+                javaDescriptor += "Z";
+            } else if (javaClass == byte.class) {
+                javaDescriptor += "B";
+            } else if (javaClass == short.class) {
+                javaDescriptor += "S";
+            } else if (javaClass == char.class) {
+                javaDescriptor += "C";
+            } else if (javaClass == int.class) {
+                javaDescriptor += "I";
+            } else if (javaClass == long.class) {
+                javaDescriptor += "J";
+            } else if (javaClass == float.class) {
+                javaDescriptor += "F";
+            } else if (javaClass == double.class) {
+                javaDescriptor += "D";
+            } else {
+                javaDescriptor += "L" + javaClass.getName() + ";";
+            }
+
+            try {
+                return Class.forName(javaDescriptor);
+            } catch (ClassNotFoundException cnfe) {
+                throw new IllegalStateException("painless type [" + painlessTypeName + "] not found", cnfe);
+            }
+        }
+
+        throw new IllegalArgumentException("painless type [" + painlessTypeName + "] not found");
+    }
+
+    public static void validatePainlessType(Class<?> painlessType, Collection<Class<?>> javaClasses) {
+        String painlessTypeName = anyTypeNameToPainlessTypeName(painlessType.getName());
+
+        while (painlessType.getComponentType() != null) {
+            painlessType = painlessType.getComponentType();
+        }
+
+        if (javaClasses.contains(painlessType) == false) {
+            throw new IllegalStateException("painless type [" + painlessTypeName + "] not found");
+        }
+    }
+
+    public static String buildPainlessMethodKey(String methodName, int methodArity) {
+        return methodName + "/" + methodArity;
+    }
+
+    public static String buildPainlessFieldKey(String fieldName) {
+        return fieldName;
+    }
+
+    public static Class<?> getBoxedAnyType(Class<?> anyType) {
+        if (anyType == boolean.class) {
+            return Boolean.class;
+        } else if (anyType == byte.class) {
+            return Byte.class;
+        } else if (anyType == short.class) {
+            return Short.class;
+        } else if (anyType == char.class) {
+            return Character.class;
+        } else if (anyType == int.class) {
+            return Integer.class;
+        } else if (anyType == long.class) {
+            return Long.class;
+        } else if (anyType == float.class) {
+            return Float.class;
+        } else if (anyType == double.class) {
+            return Double.class;
+        }
+
+        return anyType;
+    }
+
+    public static Class<?> getUnboxedAnyType(Class<?> anyType) {
+        if (anyType == Boolean.class) {
+            return boolean.class;
+        } else if (anyType == Byte.class) {
+            return byte.class;
+        } else if (anyType == Short.class) {
+            return short.class;
+        } else if (anyType == Character.class) {
+            return char.class;
+        } else if (anyType == Integer.class) {
+            return int.class;
+        } else if (anyType == Long.class) {
+            return long.class;
+        } else if (anyType == Float.class) {
+            return float.class;
+        } else if (anyType == Double.class) {
+            return double.class;
+        }
+
+        return anyType;
+    }
+
+    public static boolean isAnyTypeConstant(Class<?> anyType) {
+        return anyType == boolean.class ||
+               anyType == byte.class    ||
+               anyType == short.class   ||
+               anyType == char.class    ||
+               anyType == int.class     ||
+               anyType == long.class    ||
+               anyType == float.class   ||
+               anyType == double.class  ||
+               anyType == String.class;
+    }
+
+    public static final String DEF_PAINLESS_CLASS_NAME = def.class.getSimpleName();
+    public static final String CONSTRUCTOR_ANY_NAME = "<init>";
+
+    private PainlessLookupUtility() {
+
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
index e87e1d4bf38..8d8a7f691fe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
@@ -69,21 +69,21 @@ public class PainlessMethod {
             params = new Class<?>[1 + arguments.size()];
             params[0] = augmentation;
             for (int i = 0; i < arguments.size(); i++) {
-                params[i + 1] = PainlessLookup.defClassToObjectClass(arguments.get(i));
+                params[i + 1] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
             }
-            returnValue = PainlessLookup.defClassToObjectClass(rtn);
+            returnValue = PainlessLookupUtility.painlessDefTypeToJavaObjectType(rtn);
         } else if (Modifier.isStatic(modifiers)) {
             // static method: straightforward copy
             params = new Class<?>[arguments.size()];
             for (int i = 0; i < arguments.size(); i++) {
-                params[i] = PainlessLookup.defClassToObjectClass(arguments.get(i));
+                params[i] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
             }
-            returnValue = PainlessLookup.defClassToObjectClass(rtn);
+            returnValue = PainlessLookupUtility.painlessDefTypeToJavaObjectType(rtn);
         } else if ("<init>".equals(name)) {
             // constructor: returns the owner class
             params = new Class<?>[arguments.size()];
             for (int i = 0; i < arguments.size(); i++) {
-                params[i] = PainlessLookup.defClassToObjectClass(arguments.get(i));
+                params[i] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
             }
             returnValue = owner.clazz;
         } else {
@@ -91,9 +91,9 @@ public class PainlessMethod {
             params = new Class<?>[1 + arguments.size()];
             params[0] = owner.clazz;
             for (int i = 0; i < arguments.size(); i++) {
-                params[i + 1] = PainlessLookup.defClassToObjectClass(arguments.get(i));
+                params[i + 1] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
             }
-            returnValue = PainlessLookup.defClassToObjectClass(rtn);
+            returnValue = PainlessLookupUtility.painlessDefTypeToJavaObjectType(rtn);
         }
         return MethodType.methodType(returnValue, params);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 0dfcf4d4093..dd813f73c3d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 
 import java.util.Objects;
 
@@ -157,7 +157,7 @@ public abstract class AExpression extends ANode {
 
                 return ecast;
             } else {
-                if (PainlessLookup.isConstantType(expected)) {
+                if (PainlessLookupUtility.isAnyTypeConstant(expected)) {
                     // For the case where a cast is required, a constant is set,
                     // and the constant can be immediately cast to the expected type.
                     // An EConstant replaces this node with the constant cast appropriately
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
index a0a29ed59dd..584e5df6342 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
@@ -22,13 +22,13 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessCast;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 422300072dc..65776ca76f1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -21,14 +21,14 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.WriterConstants;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.Objects;
 import java.util.Set;
@@ -106,7 +106,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply multiply [*] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -148,7 +149,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply divide [/] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -195,7 +197,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply remainder [%] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -242,7 +245,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply add [+] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -300,7 +304,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply subtract [-] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -358,7 +363,8 @@ public final class EBinary extends AExpression {
 
         if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply left shift [<<] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote = lhspromote;
@@ -405,7 +411,8 @@ public final class EBinary extends AExpression {
 
         if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply right shift [>>] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote = lhspromote;
@@ -455,7 +462,8 @@ public final class EBinary extends AExpression {
 
         if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         if (lhspromote == def.class || rhspromote == def.class) {
@@ -498,7 +506,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply and [&] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -537,7 +546,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply xor [^] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -577,7 +587,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply or [|] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         actual = promote;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index ede1a2a6b9c..dfed0ca47b4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index c0345b6308c..e8ad9d85ed6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -21,14 +21,14 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 
@@ -69,7 +69,7 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda
                 defPointer = "D" + variable + "." + call + ",1";
             } else {
                 // typed implementation
-                defPointer = "S" + PainlessLookup.ClassToName(captured.clazz) + "." + call + ",1";
+                defPointer = "S" + PainlessLookupUtility.anyTypeToPainlessTypeName(captured.clazz) + "." + call + ",1";
             }
             actual = String.class;
         } else {
@@ -77,7 +77,8 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda
             // static case
             if (captured.clazz != def.class) {
                 try {
-                    ref = new FunctionRef(locals.getPainlessLookup(), expected, PainlessLookup.ClassToName(captured.clazz), call, 1);
+                    ref = new FunctionRef(
+                            locals.getPainlessLookup(), expected, PainlessLookupUtility.anyTypeToPainlessTypeName(captured.clazz), call, 1);
 
                     // check casts between the interface method and the delegate method are legal
                     for (int i = 0; i < ref.interfaceMethod.arguments.size(); ++i) {
@@ -109,7 +110,7 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda
             // typed interface, dynamic implementation
             writer.visitVarInsn(MethodWriter.getType(captured.clazz).getOpcode(Opcodes.ILOAD), captured.getSlot());
             Type methodType = Type.getMethodType(MethodWriter.getType(expected), MethodWriter.getType(captured.clazz));
-            writer.invokeDefCall(call, methodType, DefBootstrap.REFERENCE, PainlessLookup.ClassToName(expected));
+            writer.invokeDefCall(call, methodType, DefBootstrap.REFERENCE, PainlessLookupUtility.anyTypeToPainlessTypeName(expected));
         } else {
             // typed interface, typed implementation
             writer.visitVarInsn(MethodWriter.getType(captured.clazz).getOpcode(Opcodes.ILOAD), captured.getSlot());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index 55a9dbf71c8..b0451b685b5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 
 import java.util.Objects;
 import java.util.Set;
@@ -63,6 +63,6 @@ final class ECast extends AExpression {
 
     @Override
     public String toString() {
-        return singleLineToString(PainlessLookup.ClassToName(cast.to), child);
+        return singleLineToString(PainlessLookupUtility.anyTypeToPainlessTypeName(cast.to), child);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 806204d051a..edf18f501bc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -21,13 +21,13 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Type;
 
@@ -93,7 +93,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply equals [==] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -142,7 +143,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply reference equals [===] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         left.expected = promotedType;
@@ -182,7 +184,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply not equals [!=] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -231,7 +234,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         left.expected = promotedType;
@@ -271,7 +275,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -310,7 +315,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply greater than [>] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -349,7 +355,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -388,7 +395,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply less than [>=] to types " +
-                "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 82c24e27c5d..d4eddb059a8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -20,14 +20,14 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.objectweb.asm.Type;
 
 import java.util.Objects;
@@ -69,12 +69,12 @@ public final class EFunctionRef extends AExpression implements ILambda {
                     PainlessMethod interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod;
                     if (interfaceMethod == null) {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                                                           "to [" + PainlessLookup.ClassToName(expected) + "], not a functional interface");
+                                "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], not a functional interface");
                     }
                     PainlessMethod delegateMethod = locals.getMethod(new PainlessMethodKey(call, interfaceMethod.arguments.size()));
                     if (delegateMethod == null) {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                                                           "to [" + PainlessLookup.ClassToName(expected) + "], function not found");
+                                "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], function not found");
                     }
                     ref = new FunctionRef(expected, interfaceMethod, delegateMethod, 0);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index a3835cbc213..05564a2952e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 
 import java.util.Objects;
 import java.util.Set;
@@ -64,7 +64,8 @@ public final class EInstanceof extends AExpression {
         }
 
         // map to wrapped type for primitive types
-        resolvedType = clazz.isPrimitive() ? PainlessLookup.getBoxedType(clazz) : PainlessLookup.defClassToObjectClass(clazz);
+        resolvedType = clazz.isPrimitive() ? PainlessLookupUtility.getBoxedAnyType(clazz) :
+                PainlessLookupUtility.painlessDefTypeToJavaObjectType(clazz);
 
         // analyze and cast the expression
         expression.analyze(locals);
@@ -75,7 +76,7 @@ public final class EInstanceof extends AExpression {
         primitiveExpression = expression.actual.isPrimitive();
         // map to wrapped type for primitive types
         expressionType = expression.actual.isPrimitive() ?
-            PainlessLookup.getBoxedType(expression.actual) : PainlessLookup.defClassToObjectClass(clazz);
+            PainlessLookupUtility.getBoxedAnyType(expression.actual) : PainlessLookupUtility.painlessDefTypeToJavaObjectType(clazz);
 
         actual = boolean.class;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index 8977f4f0ef3..8e8d164b03d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -20,15 +20,15 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.node.SFunction.FunctionReserved;
 import org.objectweb.asm.Opcodes;
 
@@ -122,13 +122,13 @@ public final class ELambda extends AExpression implements ILambda {
             // we know the method statically, infer return type and any unknown/def types
             interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod;
             if (interfaceMethod == null) {
-                throw createError(new IllegalArgumentException("Cannot pass lambda to [" + PainlessLookup.ClassToName(expected) +
-                                                               "], not a functional interface"));
+                throw createError(new IllegalArgumentException("Cannot pass lambda to " +
+                        "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], not a functional interface"));
             }
             // check arity before we manipulate parameters
             if (interfaceMethod.arguments.size() != paramTypeStrs.size())
                 throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name +
-                                                   "] in [" + PainlessLookup.ClassToName(expected) + "]");
+                        "] in [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "]");
             // for method invocation, its allowed to ignore the return value
             if (interfaceMethod.rtn == void.class) {
                 returnType = def.class;
@@ -140,7 +140,7 @@ public final class ELambda extends AExpression implements ILambda {
             for (int i = 0; i < paramTypeStrs.size(); i++) {
                 String paramType = paramTypeStrs.get(i);
                 if (paramType == null) {
-                    actualParamTypeStrs.add(PainlessLookup.ClassToName(interfaceMethod.arguments.get(i)));
+                    actualParamTypeStrs.add(PainlessLookupUtility.anyTypeToPainlessTypeName(interfaceMethod.arguments.get(i)));
                 } else {
                     actualParamTypeStrs.add(paramType);
                 }
@@ -162,15 +162,15 @@ public final class ELambda extends AExpression implements ILambda {
         List<String> paramTypes = new ArrayList<>(captures.size() + actualParamTypeStrs.size());
         List<String> paramNames = new ArrayList<>(captures.size() + paramNameStrs.size());
         for (Variable var : captures) {
-            paramTypes.add(PainlessLookup.ClassToName(var.clazz));
+            paramTypes.add(PainlessLookupUtility.anyTypeToPainlessTypeName(var.clazz));
             paramNames.add(var.name);
         }
         paramTypes.addAll(actualParamTypeStrs);
         paramNames.addAll(paramNameStrs);
 
         // desugar lambda body into a synthetic method
-        desugared = new SFunction(reserved, location, PainlessLookup.ClassToName(returnType), name,
-                                            paramTypes, paramNames, statements, true);
+        desugared = new SFunction(reserved, location, PainlessLookupUtility.anyTypeToPainlessTypeName(returnType), name,
+                                  paramTypes, paramNames, statements, true);
         desugared.generateSignature(locals.getPainlessLookup());
         desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), returnType,
                                                 desugared.parameters, captures.size(), reserved.getMaxLoopCounter()));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index 820cce685ed..90475419b32 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.PainlessMethodKey;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index b6c7fb80af9..c6474846d4c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.PainlessMethodKey;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.HashMap;
 import java.util.List;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index 197e2fcb7fd..a780ea3e05b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.util.List;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 983819b6b2b..6bc5331cb1d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.objectweb.asm.Opcodes;
 
 import java.util.Set;
@@ -53,7 +53,7 @@ public final class ENull extends AExpression {
         if (expected != null) {
             if (expected.isPrimitive()) {
                 throw createError(new IllegalArgumentException(
-                    "Cannot cast null to a primitive type [" + PainlessLookup.ClassToName(expected) + "]."));
+                    "Cannot cast null to a primitive type [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "]."));
             }
 
             actual = expected;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 3a5102ebdc9..d34399db779 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -21,13 +21,13 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
@@ -93,7 +93,8 @@ public final class EUnary extends AExpression {
         promote = AnalyzerCaster.promoteNumeric(child.actual, false);
 
         if (promote == null) {
-            throw createError(new ClassCastException("Cannot apply not [~] to type [" + PainlessLookup.ClassToName(child.actual) + "]."));
+            throw createError(new ClassCastException("Cannot apply not [~] to type " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(child.actual) + "]."));
         }
 
         child.expected = promote;
@@ -122,8 +123,8 @@ public final class EUnary extends AExpression {
         promote = AnalyzerCaster.promoteNumeric(child.actual, true);
 
         if (promote == null) {
-            throw createError(
-                    new ClassCastException("Cannot apply positive [+] to type [" + PainlessLookup.ClassToName(child.actual) + "]."));
+            throw createError(new ClassCastException("Cannot apply positive [+] to type " +
+                    "[" + PainlessLookupUtility.painlessDefTypeToJavaObjectType(child.actual) + "]."));
         }
 
         child.expected = promote;
@@ -156,8 +157,8 @@ public final class EUnary extends AExpression {
         promote = AnalyzerCaster.promoteNumeric(child.actual, true);
 
         if (promote == null) {
-            throw createError(
-                    new ClassCastException("Cannot apply negative [-] to type [" + PainlessLookup.ClassToName(child.actual) + "]."));
+            throw createError(new ClassCastException("Cannot apply negative [-] to type " +
+                    "[" + PainlessLookupUtility.painlessDefTypeToJavaObjectType(child.actual) + "]."));
         }
 
         child.expected = promote;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index 5b282abdce9..c45107a37ac 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.List;
 import java.util.Map;
@@ -67,8 +67,8 @@ public final class PBrace extends AStoreable {
         } else if (List.class.isAssignableFrom(prefix.actual)) {
             sub = new PSubListShortcut(location, locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual), index);
         } else {
-            throw createError(
-                new IllegalArgumentException("Illegal array access on type [" + PainlessLookup.ClassToName(prefix.actual) + "]."));
+            throw createError(new IllegalArgumentException("Illegal array access on type " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual) + "]."));
         }
 
         sub.write = write;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index f23ae9f1887..445c053347e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -19,15 +19,15 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
-import org.elasticsearch.painless.lookup.PainlessClass;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.PainlessMethodKey;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.List;
 import java.util.Objects;
@@ -74,7 +74,7 @@ public final class PCallInvoke extends AExpression {
         PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual);
 
         if (prefix.actual.isPrimitive()) {
-            struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(PainlessLookup.getBoxedType(prefix.actual));
+            struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(PainlessLookupUtility.getBoxedAnyType(prefix.actual));
         }
 
         PainlessMethodKey methodKey = new PainlessMethodKey(name, arguments.size());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index 78a18b91ab2..3f2f8879564 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -19,16 +19,16 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessField;
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessClass;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessField;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.List;
 import java.util.Map;
@@ -64,7 +64,7 @@ public final class PField extends AStoreable {
         prefix = prefix.cast(locals);
 
         if (prefix.actual.isArray()) {
-            sub = new PSubArrayLength(location, PainlessLookup.ClassToName(prefix.actual), value);
+            sub = new PSubArrayLength(location, PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual), value);
         } else if (prefix.actual == def.class) {
             sub = new PSubDefField(location, value);
         } else {
@@ -86,7 +86,7 @@ public final class PField extends AStoreable {
                     new PainlessMethodKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
                 if (getter != null || setter != null) {
-                    sub = new PSubShortcut(location, value, PainlessLookup.ClassToName(prefix.actual), getter, setter);
+                    sub = new PSubShortcut(location, value, PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual), getter, setter);
                 } else {
                     EConstant index = new EConstant(location, value);
                     index.analyze(locals);
@@ -104,7 +104,7 @@ public final class PField extends AStoreable {
 
         if (sub == null) {
             throw createError(new IllegalArgumentException(
-                "Unknown field [" + value + "] for type [" + PainlessLookup.ClassToName(prefix.actual) + "]."));
+                "Unknown field [" + value + "] for type [" + PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual) + "]."));
         }
 
         if (nullSafe) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 66ad0ecff1b..237efa61ffa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
index ccbc25db4f2..afad497dec7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java
@@ -20,11 +20,11 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Type;
 
 import java.util.Objects;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
index a9021000e2d..7f4e253b409 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
@@ -20,11 +20,11 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Type;
 
 import java.util.ArrayList;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
index 1c081c9422e..bf00d8d8599 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
@@ -20,11 +20,11 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.Objects;
 import java.util.Set;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index d6c367cfeab..8eb154e745b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessField;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 
 import java.lang.reflect.Modifier;
 import java.util.Objects;
@@ -51,8 +51,8 @@ final class PSubField extends AStoreable {
     @Override
     void analyze(Locals locals) {
          if (write && Modifier.isFinal(field.modifiers)) {
-             throw createError(new IllegalArgumentException(
-                 "Cannot write to read-only field [" + field.name + "] for type [" + PainlessLookup.ClassToName(field.clazz) + "]."));
+             throw createError(new IllegalArgumentException("Cannot write to read-only field [" + field.name + "] for type " +
+                     "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(field.clazz) + "]."));
          }
 
         actual = field.clazz;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index 5d881b30db2..0a3ab142ddc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.WriterConstants;
+import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.util.Objects;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index 4875d55cbeb..f71e2ac5d1f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.util.Objects;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
index 4b2910dbc01..eb5668c554c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 
 import java.util.Set;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index c402d8982d8..de1a7062a24 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.def;
 
 import java.util.Objects;
 import java.util.Set;
@@ -84,8 +84,8 @@ public class SEach extends AStatement {
         } else if (expression.actual == def.class || Iterable.class.isAssignableFrom(expression.actual)) {
             sub = new SSubEachIterable(location, variable, expression, block);
         } else {
-            throw createError(
-                    new IllegalArgumentException("Illegal for each type [" + PainlessLookup.ClassToName(expression.actual) + "]."));
+            throw createError(new IllegalArgumentException("Illegal for each type " +
+                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(expression.actual) + "]."));
         }
 
         sub.analyze(locals);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 628bb1d32d5..1c801d509b5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -22,8 +22,6 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Constant;
 import org.elasticsearch.painless.Def;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Parameter;
@@ -31,6 +29,9 @@ import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.WriterConstants;
+import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.node.SSource.Reserved;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.Handle;
@@ -135,7 +136,7 @@ public final class SFunction extends AStatement {
             try {
                 Class<?> paramType = painlessLookup.getJavaClassFromPainlessType(this.paramTypeStrs.get(param));
 
-                paramClasses[param] = PainlessLookup.defClassToObjectClass(paramType);
+                paramClasses[param] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(paramType);
                 paramTypes.add(paramType);
                 parameters.add(new Parameter(location, paramNameStrs.get(param), paramType));
             } catch (IllegalArgumentException exception) {
@@ -144,8 +145,8 @@ public final class SFunction extends AStatement {
             }
         }
 
-        org.objectweb.asm.commons.Method method = new org.objectweb.asm.commons.Method(
-            name, MethodType.methodType(PainlessLookup.defClassToObjectClass(rtnType), paramClasses).toMethodDescriptorString());
+        org.objectweb.asm.commons.Method method = new org.objectweb.asm.commons.Method(name, MethodType.methodType(
+                PainlessLookupUtility.painlessDefTypeToJavaObjectType(rtnType), paramClasses).toMethodDescriptorString());
         this.method = new PainlessMethod(name, null, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index 5db161b8002..fea8c8953b6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -20,13 +20,13 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
@@ -109,6 +109,6 @@ final class SSubEachArray extends AStatement {
 
     @Override
     public String toString() {
-        return singleLineToString(PainlessLookup.ClassToName(variable.clazz), variable.name, expression, block);
+        return singleLineToString(PainlessLookupUtility.anyTypeToPainlessTypeName(variable.clazz), variable.name, expression, block);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index cfc87536b6b..cec1297a4c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -21,16 +21,16 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessCast;
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
-import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.PainlessMethodKey;
+import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
@@ -81,8 +81,8 @@ final class SSubEachIterable extends AStatement {
                     getPainlessStructFromJavaClass(expression.actual).methods.get(new PainlessMethodKey("iterator", 0));
 
             if (method == null) {
-                throw createError(new IllegalArgumentException(
-                    "Unable to create iterator for the type [" + PainlessLookup.ClassToName(expression.actual) + "]."));
+                throw createError(new IllegalArgumentException("Unable to create iterator for the type " +
+                        "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(expression.actual) + "]."));
             }
         }
 
@@ -133,6 +133,6 @@ final class SSubEachIterable extends AStatement {
 
     @Override
     public String toString() {
-        return singleLineToString(PainlessLookup.ClassToName(variable.clazz), variable.name, expression, block);
+        return singleLineToString(PainlessLookupUtility.anyTypeToPainlessTypeName(variable.clazz), variable.name, expression, block);
     }
 }

From 4b5071f2d08a2ba23eb6d84d47192453c6e68b11 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Tue, 17 Jul 2018 06:50:21 +0200
Subject: [PATCH 051/260] Add Index UUID to `/_stats` Response (#31871)

* Add "uuid" field to each index's section in the `/_stats` response
* closes #31791
---
 .../rest-api-spec/test/indices.stats/10_index.yml  |  2 ++
 .../action/admin/indices/stats/IndexStats.java     |  9 ++++++++-
 .../admin/indices/stats/IndicesStatsResponse.java  | 14 +++++++++-----
 .../admin/indices/stats/IndicesStatsTests.java     |  8 +++++++-
 .../indices/IndicesStatsMonitoringDocTests.java    |  2 +-
 5 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml
index b70fac8cf04..42847b05cd1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml
@@ -47,7 +47,9 @@ setup:
   - match: { _shards.total: 18 }
   - is_true: _all
   - is_true: indices.test1
+  - is_true: indices.test1.uuid
   - is_true: indices.test2
+  - is_true: indices.test2.uuid
 
 
 ---
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java
index 7903a340628..d8480519e5d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java
@@ -29,10 +29,13 @@ public class IndexStats implements Iterable<IndexShardStats> {
 
     private final String index;
 
+    private final String uuid;
+
     private final ShardStats shards[];
 
-    public IndexStats(String index, ShardStats[] shards) {
+    public IndexStats(String index, String uuid, ShardStats[] shards) {
         this.index = index;
+        this.uuid = uuid;
         this.shards = shards;
     }
 
@@ -40,6 +43,10 @@ public class IndexStats implements Iterable<IndexShardStats> {
         return this.index;
     }
 
+    public String getUuid() {
+        return uuid;
+    }
+
     public ShardStats[] getShards() {
         return this.shards;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
index 7406dc4f2d1..cc563948160 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -84,19 +85,22 @@ public class IndicesStatsResponse extends BroadcastResponse {
         }
         Map<String, IndexStats> indicesStats = new HashMap<>();
 
-        Set<String> indices = new HashSet<>();
+        Set<Index> indices = new HashSet<>();
         for (ShardStats shard : shards) {
-            indices.add(shard.getShardRouting().getIndexName());
+            indices.add(shard.getShardRouting().index());
         }
 
-        for (String indexName : indices) {
+        for (Index index : indices) {
             List<ShardStats> shards = new ArrayList<>();
+            String indexName = index.getName();
             for (ShardStats shard : this.shards) {
                 if (shard.getShardRouting().getIndexName().equals(indexName)) {
                     shards.add(shard);
                 }
             }
-            indicesStats.put(indexName, new IndexStats(indexName, shards.toArray(new ShardStats[shards.size()])));
+            indicesStats.put(
+                indexName, new IndexStats(indexName, index.getUUID(), shards.toArray(new ShardStats[shards.size()]))
+            );
         }
         this.indicesStats = indicesStats;
         return indicesStats;
@@ -169,7 +173,7 @@ public class IndicesStatsResponse extends BroadcastResponse {
             builder.startObject(Fields.INDICES);
             for (IndexStats indexStats : getIndices().values()) {
                 builder.startObject(indexStats.getIndex());
-
+                builder.field("uuid", indexStats.getUuid());
                 builder.startObject("primaries");
                 indexStats.getPrimaries().toXContent(builder, params);
                 builder.endObject();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
index 26785d2c870..264a92137be 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
@@ -153,6 +153,13 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
         assertEquals(0, common.refresh.getListeners());
     }
 
+    @SuppressWarnings("unchecked")
+    public void testUuidOnRootStatsIndices() {
+        String uuid = createIndex("test").indexUUID();
+        IndicesStatsResponse rsp = client().admin().indices().prepareStats().get();
+        assertEquals(uuid, rsp.getIndex("test").getUuid());
+    }
+
     /**
      * Gives access to package private IndicesStatsResponse constructor for test purpose.
      **/
@@ -160,5 +167,4 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
                                                                int failedShards, List<DefaultShardOperationFailedException> shardFailures) {
         return new IndicesStatsResponse(shards, totalShards, successfulShards, failedShards, shardFailures);
     }
-
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java
index 86fa89f4c01..66b41d40943 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java
@@ -45,7 +45,7 @@ public class IndicesStatsMonitoringDocTests extends BaseFilteredMonitoringDocTes
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        indicesStats = Collections.singletonList(new IndexStats("index-0", new ShardStats[] {
+        indicesStats = Collections.singletonList(new IndexStats("index-0", "dcvO5uZATE-EhIKc3tk9Bg", new ShardStats[] {
                 // Primaries
                 new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null),
                 new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null),

From 6717df3c2deb35ce367abedd556e337d3df7c41f Mon Sep 17 00:00:00 2001
From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com>
Date: Tue, 17 Jul 2018 15:43:03 +1000
Subject: [PATCH 052/260] [Test] Modify assert statement for ssl handshake
 (#32072)

There have been changes in error messages for `SSLHandshakeException`.
This has caused a couple of failures in our tests.
This commit modifies test verification to assert on exception type of
class `SSLHandshakeException`.
There was another issue in Java11 which caused NPE. The bug has now
been fixed on Java11 - early access build 22.
Bug Ref: https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8206355
Enable the skipped tests due to this bug.

Closes #31940
---
 .../elasticsearch/client/RestClientBuilderIntegTests.java | 8 +++-----
 .../smoketest/SmokeTestMonitoringWithSecurityIT.java      | 1 -
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
index 30359ea90f6..0d1c3ffd6b8 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
@@ -30,6 +30,7 @@ import org.junit.BeforeClass;
 
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.TrustManagerFactory;
 import java.io.IOException;
 import java.io.InputStream;
@@ -37,7 +38,7 @@ import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.security.KeyStore;
 
-import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
@@ -72,9 +73,6 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
     }
 
     public void testBuilderUsesDefaultSSLContext() throws Exception {
-        assumeFalse("Due to bug inside jdk, this test can't momentarily run with java 11. " +
-                "See: https://github.com/elastic/elasticsearch/issues/31940",
-            System.getProperty("java.version").contains("11"));
         final SSLContext defaultSSLContext = SSLContext.getDefault();
         try {
             try (RestClient client = buildRestClient()) {
@@ -82,7 +80,7 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
                     client.performRequest(new Request("GET", "/"));
                     fail("connection should have been rejected due to SSL handshake");
                 } catch (Exception e) {
-                    assertThat(e.getMessage(), containsString("General SSLEngine problem"));
+                    assertThat(e, instanceOf(SSLHandshakeException.class));
                 }
             }
 
diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
index 52c2a7dfa2d..c427d8bf32c 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
+++ b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
@@ -127,7 +127,6 @@ public class SmokeTestMonitoringWithSecurityIT extends ESIntegTestCase {
         return monitoringUsage.get().getExporters().isEmpty() == false;
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31940")
     public void testHTTPExporterWithSSL() throws Exception {
         // Ensures that the exporter is actually on
         assertBusy(() -> assertThat("[_http] exporter is not defined", getMonitoringUsageExportersDefined(), is(true)));

From 61486680a25fc16299f666385970fd7dc310271c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Tue, 17 Jul 2018 09:04:41 +0200
Subject: [PATCH 053/260] Add exclusion option to `keep_types` token filter
 (#32012)

Currently the `keep_types` token filter includes all token types specified using
its `types` parameter. Lucenes TypeTokenFilter also provides a second mode where
instead of keeping the specified tokens (include) they are filtered out
(exclude). This change exposes this option as a new `mode` parameter that can
either take the values `include` (the default, if not specified) or `exclude`.

Closes #29277
---
 .../keep-types-tokenfilter.asciidoc           | 74 ++++++++++++++++++-
 .../common/KeepTypesFilterFactory.java        | 36 +++++++--
 .../common/KeepTypesFilterFactoryTests.java   | 48 ++++++++++--
 3 files changed, 142 insertions(+), 16 deletions(-)

diff --git a/docs/reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc
index afaf4f8fa8c..05687f86691 100644
--- a/docs/reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/keep-types-tokenfilter.asciidoc
@@ -8,8 +8,9 @@ contained in a predefined set.
 [float]
 === Options
 [horizontal]
-types:: a list of types to keep
-
+types:: a list of types to include (default mode) or exclude
+mode:: if set to `include` (default) the specified token types will be kept, 
+if set to `exclude` the specified token types will be removed from the stream
 
 [float]
 === Settings example
@@ -53,7 +54,7 @@ POST /keep_types_example/_analyze
 // CONSOLE
 // TEST[continued]
 
-And it'd respond:
+The response will be:
 
 [source,js]
 --------------------------------------------------
@@ -72,3 +73,70 @@ And it'd respond:
 // TESTRESPONSE
 
 Note how only the `<NUM>` token is in the output.
+
+=== Exclude mode settings example
+
+If the `mode` parameter is set to `exclude` like in the following example:
+
+[source,js]
+--------------------------------------------------
+PUT /keep_types_exclude_example
+{
+    "settings" : {
+        "analysis" : {
+            "analyzer" : {
+                "my_analyzer" : {
+                    "tokenizer" : "standard",
+                    "filter" : ["standard", "lowercase", "remove_numbers"]
+                }
+            },
+            "filter" : {
+                "remove_numbers" : {
+                    "type" : "keep_types",
+                    "mode" : "exclude",
+                    "types" : [ "<NUM>" ]
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+And we test it like:
+
+[source,js]
+--------------------------------------------------
+POST /keep_types_exclude_example/_analyze
+{
+  "analyzer" : "my_analyzer",
+  "text" : "hello 101 world"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+The response will be:
+
+[source,js]
+--------------------------------------------------
+{
+  "tokens": [
+    {
+      "token": "hello",
+      "start_offset": 0,
+      "end_offset": 5,
+      "type": "<ALPHANUM>",
+      "position": 0
+    }, 
+    {
+      "token": "world",
+      "start_offset": 10,
+      "end_offset": 15,
+      "type": "<ALPHANUM>",
+      "position": 2
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE
\ No newline at end of file
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepTypesFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepTypesFilterFactory.java
index 0f94b521e4b..b6b8b45fabf 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepTypesFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeepTypesFilterFactory.java
@@ -29,21 +29,47 @@ import org.elasticsearch.index.analysis.TokenFilterFactory;
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 
 /**
  * A {@link TokenFilterFactory} for {@link TypeTokenFilter}. This filter only
  * keep tokens that are contained in the set configured via
- * {@value #KEEP_TYPES_KEY} setting.
+ * {@value #KEEP_TYPES_MODE_KEY} setting.
  * <p>
  * Configuration options:
  * <ul>
- * <li>{@value #KEEP_TYPES_KEY} the array of words / tokens to keep.</li>
+ * <li>{@value #KEEP_TYPES_KEY} the array of words / tokens.</li>
+ * <li>{@value #KEEP_TYPES_MODE_KEY} whether to keep ("include") or discard
+ * ("exclude") the specified token types.</li>
  * </ul>
  */
 public class KeepTypesFilterFactory extends AbstractTokenFilterFactory {
     private final Set<String> keepTypes;
-    private static final String KEEP_TYPES_KEY = "types";
+    private final KeepTypesMode includeMode;
+    static final String KEEP_TYPES_KEY = "types";
+    static final String KEEP_TYPES_MODE_KEY = "mode";
+
+    enum KeepTypesMode {
+        INCLUDE, EXCLUDE;
+
+        @Override
+        public String toString() {
+            return this.name().toLowerCase(Locale.ROOT);
+        }
+
+        private static KeepTypesMode fromString(String modeString) {
+            String lc = modeString.toLowerCase(Locale.ROOT);
+            if (lc.equals("include")) {
+                return INCLUDE;
+            } else if (lc.equals("exclude")) {
+                return EXCLUDE;
+            } else {
+                throw new IllegalArgumentException("`keep_types` tokenfilter mode can only be [" + KeepTypesMode.INCLUDE + "] or ["
+                        + KeepTypesMode.EXCLUDE + "] but was [" + modeString + "].");
+            }
+        }
+    };
 
     KeepTypesFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -52,12 +78,12 @@ public class KeepTypesFilterFactory extends AbstractTokenFilterFactory {
         if ((arrayKeepTypes == null)) {
             throw new IllegalArgumentException("keep_types requires `" + KEEP_TYPES_KEY + "` to be configured");
         }
-
+        this.includeMode = KeepTypesMode.fromString(settings.get(KEEP_TYPES_MODE_KEY, "include"));
         this.keepTypes = new HashSet<>(arrayKeepTypes);
     }
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
-        return new TypeTokenFilter(tokenStream, keepTypes, true);
+        return new TypeTokenFilter(tokenStream, keepTypes, includeMode == KeepTypesMode.INCLUDE);
     }
 }
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepTypesFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepTypesFilterFactoryTests.java
index a19882d6faa..d0c7723457f 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepTypesFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/KeepTypesFilterFactoryTests.java
@@ -34,19 +34,51 @@ import java.io.StringReader;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class KeepTypesFilterFactoryTests extends ESTokenStreamTestCase {
-    public void testKeepTypes() throws IOException {
-        Settings settings = Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .put("index.analysis.filter.keep_numbers.type", "keep_types")
-                .putList("index.analysis.filter.keep_numbers.types", new String[] {"<NUM>", "<SOMETHINGELSE>"})
-                .build();
+
+    private static final String BASE_SETTING = "index.analysis.filter.keep_numbers";
+
+    public void testKeepTypesInclude() throws IOException {
+        Settings.Builder settingsBuilder = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(BASE_SETTING + ".type", "keep_types")
+                .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "<NUM>", "<SOMETHINGELSE>" });
+        // either use default mode or set "include" mode explicitly
+        if (random().nextBoolean()) {
+            settingsBuilder.put(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY,
+                    KeepTypesFilterFactory.KeepTypesMode.INCLUDE);
+        }
+        Settings settings = settingsBuilder.build();
         ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
         TokenFilterFactory tokenFilter = analysis.tokenFilter.get("keep_numbers");
         assertThat(tokenFilter, instanceOf(KeepTypesFilterFactory.class));
         String source = "Hello 123 world";
-        String[] expected = new String[]{"123"};
+        String[] expected = new String[] { "123" };
         Tokenizer tokenizer = new StandardTokenizer();
         tokenizer.setReader(new StringReader(source));
-        assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[]{2});
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[] { 2 });
+    }
+
+    public void testKeepTypesExclude() throws IOException {
+        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(BASE_SETTING + ".type", "keep_types")
+                .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "<NUM>", "<SOMETHINGELSE>" })
+                .put(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY, KeepTypesFilterFactory.KeepTypesMode.EXCLUDE).build();
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
+        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("keep_numbers");
+        assertThat(tokenFilter, instanceOf(KeepTypesFilterFactory.class));
+        String source = "Hello 123 world";
+        String[] expected = new String[] { "Hello", "world" };
+        Tokenizer tokenizer = new StandardTokenizer();
+        tokenizer.setReader(new StringReader(source));
+        assertTokenStreamContents(tokenFilter.create(tokenizer), expected, new int[] { 1, 2 });
+    }
+
+    public void testKeepTypesException() throws IOException {
+        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(BASE_SETTING + ".type", "keep_types")
+                .putList(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_KEY, new String[] { "<NUM>", "<SOMETHINGELSE>" })
+                .put(BASE_SETTING + "." + KeepTypesFilterFactory.KEEP_TYPES_MODE_KEY, "bad_parameter").build();
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin()));
+        assertEquals("`keep_types` tokenfilter mode can only be [include] or [exclude] but was [bad_parameter].", ex.getMessage());
     }
 }

From 36165265ce6010a33147a6e3c5296a64f989944d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Tue, 17 Jul 2018 09:09:03 +0200
Subject: [PATCH 054/260] Fix put mappings java API documentation (#31955)

The current docs of the put-mapping Java API is currently broken. It its current
form, it creates an index and uses the whole mapping definition given as a JSON
string as the type name. Since we didn't check the index created in the
IndicesDocumentationIT so far this went unnoticed.

This change adds test to catch this error to the documentation test, changes the
documentation so it works correctly now and adds an input validation to
PutMappingRequest#buildFromSimplifiedDef() which was used internally to reject
calls where no mapping definition is given.

Closes #31906
---
 .../admin/indices/put-mapping.asciidoc        | 11 ++++--
 .../mapping/put/PutMappingRequest.java        | 11 +++---
 .../mapping/put/PutMappingRequestTests.java   |  6 +++-
 .../documentation/IndicesDocumentationIT.java | 35 ++++++++++++++-----
 4 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/docs/java-api/admin/indices/put-mapping.asciidoc b/docs/java-api/admin/indices/put-mapping.asciidoc
index 3e931dfd7b7..8bdcb491697 100644
--- a/docs/java-api/admin/indices/put-mapping.asciidoc
+++ b/docs/java-api/admin/indices/put-mapping.asciidoc
@@ -2,17 +2,22 @@
 
 ==== Put Mapping
 
-The PUT mapping API allows you to add a new type while creating an index:
+You can add mappings for a new type at index creation time:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
 include-tagged::{client-tests}/IndicesDocumentationIT.java[index-with-mapping]
 --------------------------------------------------
 <1> <<java-admin-indices-create-index,Creates an index>> called `twitter`
-<2> It also adds a `tweet` mapping type.
+<2> Add a `tweet` type with a field called `message` that has the datatype `text`.
 
+There are several variants of the above `addMapping` method, some taking an
+`XContentBuilder` or a `Map` with the mapping definition as arguments. Make sure
+to check the javadocs to pick the simplest one for your use case.
 
-The PUT mapping API also allows to add a new type to an existing index:
+The PUT mapping API also allows to specify the mapping of a type after index
+creation. In this case you can provide the mapping as a String similar to the
+Rest API syntax:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
index dc201b38c3b..3429b35073c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -184,10 +184,13 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
     }
 
     /**
-     * @param type the mapping type
-     * @param source consisting of field/properties pairs (e.g. "field1",
-     *            "type=string,store=true"). If the number of arguments is not
-     *            divisible by two an {@link IllegalArgumentException} is thrown
+     * @param type
+     *            the mapping type
+     * @param source
+     *            consisting of field/properties pairs (e.g. "field1",
+     *            "type=string,store=true")
+     * @throws IllegalArgumentException
+     *             if the number of the source arguments is not divisible by two
      * @return the mappings definition
      */
     public static XContentBuilder buildFromSimplifiedDef(String type, Object... source) {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
index be44d790b40..86c2b67be9c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
@@ -76,8 +76,12 @@ public class PutMappingRequestTests extends ESTestCase {
                 " concrete index: [[foo/bar]] and indices: [myindex];");
     }
 
+    /**
+     * Test that {@link PutMappingRequest#buildFromSimplifiedDef(String, Object...)}
+     * rejects inputs where the {@code Object...} varargs of field name and properties are not
+     * paired correctly
+     */
     public void testBuildFromSimplifiedDef() {
-        // test that method rejects input where input varargs fieldname/properites are not paired correctly
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> PutMappingRequest.buildFromSimplifiedDef("type", "only_field"));
         assertEquals("mapping source must be pairs of fieldnames and properties definition.", e.getMessage());
diff --git a/server/src/test/java/org/elasticsearch/client/documentation/IndicesDocumentationIT.java b/server/src/test/java/org/elasticsearch/client/documentation/IndicesDocumentationIT.java
index 064702170d5..e5df229cd98 100644
--- a/server/src/test/java/org/elasticsearch/client/documentation/IndicesDocumentationIT.java
+++ b/server/src/test/java/org/elasticsearch/client/documentation/IndicesDocumentationIT.java
@@ -19,10 +19,19 @@
 
 package org.elasticsearch.client.documentation;
 
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.instanceOf;
+
 /**
  * This class is used to generate the Java indices administration documentation.
  * You need to wrap your code between two tags like:
@@ -48,16 +57,14 @@ public class IndicesDocumentationIT extends ESIntegTestCase {
         Client client = client();
 
         // tag::index-with-mapping
-        client.admin().indices().prepareCreate("twitter")   // <1>
-                .addMapping("\"tweet\": {\n" +              // <2>
-                        "  \"properties\": {\n" +
-                        "    \"message\": {\n" +
-                        "      \"type\": \"text\"\n" +
-                        "    }\n" +
-                        "  }\n" +
-                        "}")
+        client.admin().indices().prepareCreate("twitter")    // <1>
+                .addMapping("tweet", "message", "type=text") // <2>
                 .get();
         // end::index-with-mapping
+        GetMappingsResponse getMappingsResponse = client.admin().indices().prepareGetMappings("twitter").get();
+        assertEquals(1, getMappingsResponse.getMappings().size());
+        ImmutableOpenMap<String, MappingMetaData> indexMapping = getMappingsResponse.getMappings().get("twitter");
+        assertThat(indexMapping.get("tweet"), instanceOf(MappingMetaData.class));
 
         // we need to delete in order to create a fresh new index with another type
         client.admin().indices().prepareDelete("twitter").get();
@@ -89,6 +96,11 @@ public class IndicesDocumentationIT extends ESIntegTestCase {
                 "}", XContentType.JSON)
         .get();
         // end::putMapping-request-source
+        getMappingsResponse = client.admin().indices().prepareGetMappings("twitter").get();
+        assertEquals(1, getMappingsResponse.getMappings().size());
+        indexMapping = getMappingsResponse.getMappings().get("twitter");
+        assertEquals(singletonMap("properties", singletonMap("name", singletonMap("type", "text"))),
+                indexMapping.get("user").getSourceAsMap());
 
         // tag::putMapping-request-source-append
         client.admin().indices().preparePutMapping("twitter")   // <1>
@@ -102,6 +114,13 @@ public class IndicesDocumentationIT extends ESIntegTestCase {
                 "}", XContentType.JSON)
         .get();
         // end::putMapping-request-source-append
+        getMappingsResponse = client.admin().indices().prepareGetMappings("twitter").get();
+        assertEquals(1, getMappingsResponse.getMappings().size());
+        indexMapping = getMappingsResponse.getMappings().get("twitter");
+        Map<String, Map<?,?>> expected = new HashMap<>();
+        expected.put("name", singletonMap("type", "text"));
+        expected.put("user_name", singletonMap("type", "text"));
+        assertEquals(expected, indexMapping.get("user").getSourceAsMap().get("properties"));
     }
 
 }

From 9e529d9d584ae6f6978ac09056b73d9ccdde4a3c Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ikakavas@protonmail.com>
Date: Tue, 17 Jul 2018 10:54:10 +0300
Subject: [PATCH 055/260] Enable testing in FIPS140 JVM (#31666)

Ensure our tests can run in a FIPS JVM

JKS keystores cannot be used in a FIPS JVM as attempting to use one
in order to init a KeyManagerFactory or a TrustManagerFactory is not
allowed.( JKS keystore algorithms for private key encryption are not
FIPS 140 approved)
This commit replaces JKS keystores in our tests with the
corresponding PEM encoded key and certificates both for key and trust
configurations.
Whenever it's not possible to refactor the test, i.e. when we are
testing that we can load a JKS keystore, etc. we attempt to
mute the test when we are running in FIPS 140 JVM. Testing for the
JVM is naive and is based on the name of the security provider as
we would control the testing infrastrtucture and so this would be
reliable enough.
Other cases of tests being muted are the ones that involve custom
TrustStoreManagers or KeyStoreManagers, null TLS Ciphers and the
SAMLAuthneticator class as we cannot sign XML documents in the
way we were doing. SAMLAuthenticator tests in a FIPS JVM can be
reenabled with precomputed and signed SAML messages at a later stage.

IT will be covered in a subsequent PR
---
 client/rest/build.gradle                      |   4 +
 .../client/RestClientBuilderIntegTests.java   |  24 ++-
 client/rest/src/test/resources/test.crt       |  24 +++
 client/rest/src/test/resources/test.der       | Bin 0 -> 1218 bytes
 .../plugins/InstallPluginCommandTests.java    |   6 +
 .../ingest/common/DateProcessorTests.java     |   2 +
 .../AzureDiscoveryClusterFormationTests.java  |  10 ++
 .../common/settings/KeyStoreWrapperTests.java |   2 +
 .../org/elasticsearch/test/ESTestCase.java    |   6 +
 .../xpack/core/ssl/CertParsingUtils.java      |   4 +-
 .../license/LicenseServiceClusterTests.java   |   1 +
 .../license/SelfGeneratedLicenseTests.java    |   1 +
 .../xpack/core/ssl/CertGenUtilsTests.java     |   6 +
 .../xpack/core/ssl/PemUtilsTests.java         |   1 +
 .../ssl/SSLConfigurationReloaderTests.java    |  94 +++++++----
 .../xpack/core/ssl/SSLServiceTests.java       |  92 ++++++-----
 .../xpack/core/ssl/StoreKeyConfigTests.java   |   2 +
 .../transport/ssl/certs/simple/samba4.crt     |  22 +++
 .../ssl/certs/simple/testnode-ip-only.pem     |  30 ++++
 ...tname.cert => testnode-no-subjaltname.crt} |   0
 .../certs/simple/testnode-no-subjaltname.pem  |  30 ++++
 .../ssl/certs/simple/testnode_updated.crt     |  49 +++---
 .../ssl/certs/simple/testnode_updated.jks     | Bin 2333 -> 4003 bytes
 .../ssl/certs/simple/testnode_updated.p12     | Bin 2628 -> 4300 bytes
 .../ssl/certs/simple/testnode_updated.pem     |  77 ++++++---
 .../test/SecuritySettingsSource.java          |  27 +++-
 .../test/SettingsFilterTests.java             |  13 +-
 .../security/PkiRealmBootstrapCheckTests.java |  20 +--
 .../esnative/ESNativeMigrateToolTests.java    |  16 +-
 .../tool/CommandLineHttpClientTests.java      |  23 ++-
 .../authc/ldap/GroupsResolverTestCase.java    |   4 +-
 .../security/authc/ldap/LdapTestUtils.java    |   2 +-
 .../LdapUserSearchSessionFactoryTests.java    |   9 +-
 .../authc/pki/PkiAuthenticationTests.java     |  75 +++++----
 .../authc/saml/SamlAuthenticatorTests.java    |   4 +-
 .../authc/saml/SamlMetadataCommandTests.java  |  14 +-
 .../security/authc/saml/SamlRealmTests.java   |  40 +++--
 ...ServerTransportFilterIntegrationTests.java |  46 +++---
 .../netty4/IPHostnameVerificationTests.java   |  40 ++---
 ...ecurityNetty4HttpServerTransportTests.java |  42 ++---
 .../SecurityNetty4ServerTransportTests.java   |  37 +++--
 .../netty4/SslHostnameVerificationTests.java  |  55 ++++---
 .../transport/nio/SSLDriverTests.java         |  31 ++--
 .../nio/SimpleSecurityNioTransportTests.java  |  16 +-
 .../transport/ssl/EllipticCurveSSLTests.java  |   4 +-
 .../transport/ssl/SslIntegrationTests.java    |  11 +-
 .../transport/ssl/SslMultiPortTests.java      | 153 +++++++++---------
 .../transport/ssl/SslNullCipherTests.java     |   6 +
 .../xpack/ssl/SSLClientAuthTests.java         |  65 ++++----
 .../xpack/ssl/SSLReloadIntegTests.java        | 103 ++++++------
 .../xpack/ssl/SSLTrustRestrictionsTests.java  |   1 +
 .../xpack/security/authc/ldap/support/ad.crt  |  23 +++
 .../security/authc/ldap/support/smb_ca.crt    |  22 +++
 .../security/authc/ldap/support/smb_cert.crt  |  22 +++
 .../webhook/WebhookHttpsIntegrationTests.java |  12 +-
 .../watcher/common/http/HttpClientTests.java  | 100 ++++++------
 .../xpack/security/keystore/testnode.crt      |  21 +++
 .../xpack/security/keystore/testnode.pem      |  27 ++++
 .../keystore/truststore-testnode-only.crt     |  21 +++
 .../certs/simple/testnode-no-subjaltname.crt  |  20 +++
 .../certs/simple/testnode-no-subjaltname.pem  |  30 ++++
 x-pack/qa/openldap-tests/build.gradle         |   1 +
 ...OpenLdapUserSearchSessionFactoryTests.java |  14 +-
 .../authc/ldap/SearchGroupsResolverTests.java |   2 +-
 .../ssl/CertificateGenerateToolTests.java     |   6 +
 .../xpack/core/ssl/CertificateToolTests.java  |   6 +
 .../ADLdapUserSearchSessionFactoryTests.java  |   9 +-
 .../ldap/AbstractActiveDirectoryTestCase.java |  38 +++--
 .../ldap/AbstractAdLdapRealmTestCase.java     |  76 ++++++---
 .../ActiveDirectoryGroupsResolverTests.java   |   2 +-
 .../ActiveDirectorySessionFactoryTests.java   |  24 ++-
 .../authc/ldap/MultipleAdRealmIT.java         |   5 +-
 72 files changed, 1215 insertions(+), 610 deletions(-)
 create mode 100644 client/rest/src/test/resources/test.crt
 create mode 100644 client/rest/src/test/resources/test.der
 create mode 100644 x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt
 create mode 100644 x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.pem
 rename x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/{testnode-no-subjaltname.cert => testnode-no-subjaltname.crt} (100%)
 create mode 100644 x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem
 create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ad.crt
 create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_ca.crt
 create mode 100644 x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_cert.crt
 create mode 100644 x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/testnode.crt
 create mode 100644 x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/testnode.pem
 create mode 100644 x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/truststore-testnode-only.crt
 create mode 100644 x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt
 create mode 100644 x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem

diff --git a/client/rest/build.gradle b/client/rest/build.gradle
index b1ed05a8342..fc2ab0bc4c0 100644
--- a/client/rest/build.gradle
+++ b/client/rest/build.gradle
@@ -59,6 +59,10 @@ forbiddenApisMain {
                     PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
 }
 
+forbiddenPatterns {
+  exclude '**/*.der'
+}
+
 forbiddenApisTest {
   //we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
   bundledSignatures -= 'jdk-non-portable'
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
index 0d1c3ffd6b8..49eefc527ba 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
@@ -36,7 +36,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.KeyFactory;
 import java.security.KeyStore;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.security.spec.PKCS8EncodedKeySpec;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertEquals;
@@ -101,12 +107,20 @@ public class RestClientBuilderIntegTests extends RestClientTestCase {
 
     private static SSLContext getSslContext() throws Exception {
         SSLContext sslContext = SSLContext.getInstance("TLS");
-        try (InputStream in = RestClientBuilderIntegTests.class.getResourceAsStream("/testks.jks")) {
-            KeyStore keyStore = KeyStore.getInstance("JKS");
-            keyStore.load(in, "password".toCharArray());
-            KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
+        try (InputStream certFile = RestClientBuilderIntegTests.class.getResourceAsStream("/test.crt")) {
+            // Build a keystore of default type programmatically since we can't use JKS keystores to
+            // init a KeyManagerFactory in FIPS 140 JVMs.
+            KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+            keyStore.load(null, "password".toCharArray());
+            CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
+            PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(Files.readAllBytes(Paths.get(RestClientBuilderIntegTests.class
+                .getResource("/test.der").toURI())));
+            KeyFactory keyFactory = KeyFactory.getInstance("RSA");
+            keyStore.setKeyEntry("mykey", keyFactory.generatePrivate(privateKeySpec), "password".toCharArray(),
+                new Certificate[]{certFactory.generateCertificate(certFile)});
+            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
             kmf.init(keyStore, "password".toCharArray());
-            TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
+            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
             tmf.init(keyStore);
             sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
         }
diff --git a/client/rest/src/test/resources/test.crt b/client/rest/src/test/resources/test.crt
new file mode 100644
index 00000000000..ab1b8c2265d
--- /dev/null
+++ b/client/rest/src/test/resources/test.crt
@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEATCCAumgAwIBAgIEObhDZDANBgkqhkiG9w0BAQsFADBnMQswCQYDVQQGEwJV
+UzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxEDAOBgNVBAoT
+B2VsYXN0aWMxDTALBgNVBAsTBHRlc3QxEjAQBgNVBAMTCXRlc3Qgbm9kZTAeFw0x
+NzA3MTcxNjEyNTZaFw0yNzA3MTUxNjEyNTZaMGcxCzAJBgNVBAYTAlVTMQswCQYD
+VQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4GA1UEChMHZWxhc3Rp
+YzENMAsGA1UECxMEdGVzdDESMBAGA1UEAxMJdGVzdCBub2RlMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnXtuGIgAq6vWzUD34HXkYF+0u103hb8d1h35
+kjeuNApkUhS6x/VbuNp7TpWmprfDgG5w9TourHvyiqcQMDEWrBunS6rmKo1jK1Wm
+le3qA3F2l9VIZSNeeYQgezmzuElEPPmBjN8XBByIWKYjZcGd5u7DiquPUh9QLIev
+itgB2jfi9D8ewyvaSbVAQuQwyIaDN9L74wKyMC8EuzzAWNSDjgIhhwcR5qg17msa
+ItyM44/3hik+ObIGpMlLSxQu2V1U9bOaq48JjQBLHVg1vzC9VzGuNdEb8haFnhJN
+UrdESdHymbtBSUvy30iB+kHq5R8wQ4pC+WxChQnbA2GskuFrMQIDAQABo4G0MIGx
+MIGPBgNVHREEgYcwgYSHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCCWxvY2FsaG9z
+dIIVbG9jYWxob3N0LmxvY2FsZG9tYWluggpsb2NhbGhvc3Q0ghdsb2NhbGhvc3Q0
+LmxvY2FsZG9tYWluNIIKbG9jYWxob3N0NoIXbG9jYWxob3N0Ni5sb2NhbGRvbWFp
+bjYwHQYDVR0OBBYEFFwNcqIKfGBCBGo9faQJ3TsHmp0SMA0GCSqGSIb3DQEBCwUA
+A4IBAQBvUJTRjSOf/+vtyS3OokwRilg1ZGF3psg0DWhjH2ehIRfNibU1Y8FVQo3I
+VU8LjcIUK1cN85z+AsYqLXo/C4qmJPydQ1tGpQL7uIrPD4h+Xh3tY6A2DKRJRQFO
+w2LjswPidGufMztpPbXxLREqvkvn80VkDnc44UPxYfHvZFqYwYyxZccA5mm+BhYu
+IerjfvgX+8zMWIQZOd+jRq8EaVTmVK2Azwwhc5ImWfc0DA3pmGPdECzE4N0VVoIJ
+N8PCVltXXP3F7K3LoT6CLSiJ3c/IDVNoVS4pRV6R6Y4oIKD9T/T1kAgAvOrUGRWY
+ejWQ41GdUmkmxrqCaMbVCO4s72BC
+-----END CERTIFICATE-----
diff --git a/client/rest/src/test/resources/test.der b/client/rest/src/test/resources/test.der
new file mode 100644
index 0000000000000000000000000000000000000000..454bfd286bd97937020cb8d1535c7cb3eec43557
GIT binary patch
literal 1218
zcmV;z1U>sOf&{(-0RS)!1_>&LNQU<f0RaI800e>rs4#*Aqyhl|0)hbn0G)eo7>EF?
ztJcjx_uzHpU|+PmT{nfl9o8NBk~gk23S?3gy2tffxY~P8m8Pb*!+>sZ^*S!Bd-95>
z5HK+otQ)6Gs^%(<V=GmrmF?;Sadwy0NM$2ld4wQ)IkUJ)L_GO{jNcao9Eez^BW1yz
z=I+CatB+D2P%MY9ir4|#H{$d^9>XizNwq*i<S@vFgE!Lq;{viUF9f?hz*y9SjshWv
z2NC9|HSTK~BHWDQkN1WtJ~^@mq{&N56fW6aRQ0o(tB(nd081TMHNP;uS23<N(Hrs>
zg`N^kQny4&(ejzQK}k#U-$;S_LF(lnFhhz$`D{Xk3EKl<tdiktF#-bt009Dm0RS`-
zY8M7^Vp36-P}3!k71E#*yqQ=Ki(f1|ff!Bv_6#wb%4+aWI^yQ5I}+5T@4;UvC54cc
zdPfPZy9?<<asj%l7^DlcDoGAKCH4_g{fv0<KYz}lVNJl;l%_$vzs)9eLD%@SJ(0<)
zwo{@5;aEt%vyFndUd}zsC(1h>ajxI^3zY*DMf42|U+NcycGN5Ii(&fd7Vo1LaD{Jj
zk^&xUtM1`nB%&;_e1ejavA5F)$JNV-kx=7#<u`H2+^F3%yv%?da8Hv8l;0OEzO&78
zB`r6aGjg_EqFYCu#hJNy-ONKA;!aXOySIM$$y5X{%<Wa!u3Z<T0bhM1YdHdefdJML
z3_Y*<A6E-_+Ij!hw)jcv;)=RgAeS<@ae9dwqCk9yVC~Rlt`HYoK<J(fwM=Ab*>3=1
zNR_IxrXGQvev4;>7K$A&AgJXrpFZ9tBu-XU3Uf%X^y(KqMsL0Z-C#fGDTJnM2BQ?+
zKE>d@a-CM0qU0BWzEYKKBGDEJ^dka+fdITy>*NbhtzpqD5?=qK)<@mW-;493$$ORU
z8{^b}t5lA~o?5U70h1-GV?nHhZXY@>$0xT2_4l9h5S-b#TRao!cqSaPrDKxeqJOjo
zml{5yQNEYHnnJMOUHUBmo4Q=cM1qpnio#8Uxx~|?($N?p><9S#EFe*7KU@iKo|^)J
zfdGPA6Co{I{$ZaM#M7<2E=U`i@W9cZ<g~4-D^M2IB_L<Cc#YU?QiXB!H-c|m1-i2u
z(a{P<&op##y*{pI9ee&+EbxWB0RmVq({2$EME`#Z;s!aXs5&%Mm*dN>Vz3Rl(n@%p
zC!K#o3d*d6Fq<ay|9~yvPw6ro_bQ`h1W5vcfFm()n2P|FzWmO{lo%tf%y~!3fThF=
z$J~jPx2mH(3N0`K+M%GWVyPT)4Ee`7sQy27nipE;rXsGQsvQZHixH7jx{{28u&U=B
zZOz}AXCEUNT8xt7(&?zAFgGr+>rbI!Bq4gl1aW28R!`EhPQJRp7x7Sbl81yUf$@cT
zR|0{70Hk{CRiE_&-^6Tw?fk}J`Eyu<XV;)*?RBzw!?9s%S8TOcr5`<U$S$oQ8hL+<
zd^)|e5%HnKJIZ;T6t%M2G62jv5o0OXS1LgpQI&_#Qi`Sn{_GdEuMGY60YI4ZG8-(b
gIpzcvNC^2Pf3Si{z=fQGW<+iLqmUHJ77O9_M+?_L1ONa4

literal 0
HcmV?d00001

diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index 1db551934c7..ceff354f15f 100644
--- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -61,6 +61,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.PosixPermissionsResetter;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
@@ -139,6 +140,11 @@ public class InstallPluginCommandTests extends ESTestCase {
         System.setProperty("java.io.tmpdir", temp.apply("tmpdir").toString());
     }
 
+    @BeforeClass
+    public static void testIfFipsMode() {
+        assumeFalse("Can't run in a FIPS JVM because this depends on BouncyCastle (non-fips)", inFipsJvm());
+    }
+
     @Override
     @Before
     public void setUp() throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
index 43a5f9245b1..23aac797859 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
@@ -114,6 +114,8 @@ public class DateProcessorTests extends ESTestCase {
     }
 
     public void testJodaPatternLocale() {
+        //TODO investigate if this is a bug in Joda
+        assumeFalse("Can't run in a FIPS JVM, Joda parse date error", inFipsJvm());
             DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
             templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ITALIAN),
                 "date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date");
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
index 1b8ca38aec4..c5a5f1df98e 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
@@ -40,6 +40,8 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.transport.TcpTransport;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.ExternalResource;
 
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
@@ -87,6 +89,14 @@ public class AzureDiscoveryClusterFormationTests extends ESIntegTestCase {
 
     private static Path keyStoreFile;
 
+    @ClassRule
+    public static final ExternalResource MUTE_IN_FIPS_JVM = new ExternalResource() {
+        @Override
+        protected void before() {
+            assumeFalse("Can't run in a FIPS JVM because none of the supported Keystore types can be used", inFipsJvm());
+        }
+    };
+
     @BeforeClass
     public static void setupKeyStore() throws IOException {
         Path tempDir = createTempDir();
diff --git a/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
index 849841943ec..fe7b02d63ec 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
@@ -290,6 +290,7 @@ public class KeyStoreWrapperTests extends ESTestCase {
     }
 
     public void testBackcompatV1() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
         Path configDir = env.configFile();
         SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
         try (IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) {
@@ -320,6 +321,7 @@ public class KeyStoreWrapperTests extends ESTestCase {
     }
 
     public void testBackcompatV2() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
         Path configDir = env.configFile();
         SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
         byte[] fileBytes = new byte[20];
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 9cdfc6776f8..4c00527a932 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -125,6 +125,7 @@ import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.ZoneId;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -132,6 +133,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Random;
@@ -1363,4 +1365,8 @@ public abstract class ESTestCase extends LuceneTestCase {
         }
     }
 
+    public static boolean inFipsJvm() {
+        return Security.getProviders()[0].getName().toLowerCase(Locale.ROOT).contains("fips");
+    }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
index 8f66af14dfc..1617a92b550 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
@@ -158,7 +158,7 @@ public class CertParsingUtils {
 
     private static KeyStore getKeyStore(Certificate[] certificateChain, PrivateKey privateKey, char[] password)
             throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
-        KeyStore keyStore = KeyStore.getInstance("jks");
+        KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
         keyStore.load(null, null);
         // password must be non-null for keystore...
         keyStore.setKeyEntry("key", privateKey, password, certificateChain);
@@ -242,7 +242,7 @@ public class CertParsingUtils {
     static KeyStore trustStore(Certificate[] certificates)
             throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
         assert certificates != null : "Cannot create trust store with null certificates";
-        KeyStore store = KeyStore.getInstance("jks");
+        KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
         store.load(null, null);
         int counter = 0;
         for (Certificate certificate : certificates) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java
index a58491dea3b..00d1c47cded 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java
@@ -153,6 +153,7 @@ public class LicenseServiceClusterTests extends AbstractLicensesIntegrationTestC
     }
 
     public void testClusterRestartWithOldSignature() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM. We can't generate old licenses since PBEWithSHA1AndDESede is not available", inFipsJvm());
         wipeAllLicenses();
         internalCluster().startNode();
         ensureGreen();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/SelfGeneratedLicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/SelfGeneratedLicenseTests.java
index 4e061623ccd..0b61604ed7c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/SelfGeneratedLicenseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/SelfGeneratedLicenseTests.java
@@ -76,6 +76,7 @@ public class SelfGeneratedLicenseTests extends ESTestCase {
     }
 
     public void testTrialLicenseVerifyWithOlderVersion() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM. We can't generate old licenses since PBEWithSHA1AndDESede is not available", inFipsJvm());
         long issueDate = System.currentTimeMillis();
         License.Builder specBuilder = License.builder()
                 .issuedTo("customer")
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertGenUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertGenUtilsTests.java
index cf6ab53b9f5..20259144b42 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertGenUtilsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertGenUtilsTests.java
@@ -12,6 +12,7 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
 
 import java.math.BigInteger;
 import java.net.InetAddress;
@@ -32,6 +33,11 @@ import static org.mockito.Mockito.when;
  */
 public class CertGenUtilsTests extends ESTestCase {
 
+    @BeforeClass
+    public static void muteInFips(){
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
+    }
+
     public void testSerialNotRepeated() {
         int iterations = scaledRandomIntBetween(10, 100);
         List<BigInteger> list = new ArrayList<>(iterations);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/PemUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/PemUtilsTests.java
index 880cb03a64f..b82275a8833 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/PemUtilsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/PemUtilsTests.java
@@ -53,6 +53,7 @@ public class PemUtilsTests extends ESTestCase {
     }
 
     public void testReadEncryptedPKCS8Key() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, PBE KeySpec is not available", inFipsJvm());
         Key key = getKeyFromKeystore("RSA");
         assertThat(key, notNullValue());
         assertThat(key, instanceOf(PrivateKey.class));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
index 72cd13471df..d8e0b693f70 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
@@ -42,7 +42,10 @@ import java.security.NoSuchAlgorithmException;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.security.UnrecoverableKeyException;
+import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.function.Consumer;
 
@@ -76,6 +79,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
      * Tests reloading a keystore that is used in the KeyManager of SSLContext
      */
     public void testReloadingKeyStore() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         final Path tempDir = createTempDir();
         final Path keystorePath = tempDir.resolve("testnode.jks");
         final Path updatedKeystorePath = tempDir.resolve("testnode_updated.jks");
@@ -133,12 +137,10 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
         Path updatedKeyPath = tempDir.resolve("testnode_updated.pem");
         Path certPath = tempDir.resolve("testnode.crt");
         Path updatedCertPath = tempDir.resolve("testnode_updated.crt");
-        final Path clientTruststorePath = tempDir.resolve("testnode.jks");
         Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"), keyPath);
         Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem"), updatedKeyPath);
         Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt"), updatedCertPath);
         Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"), certPath);
-        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"), clientTruststorePath);
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         final Settings settings = Settings.builder()
@@ -150,7 +152,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
         final Environment env = randomBoolean() ? null :
             TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         // Load HTTPClient once. Client uses a keystore containing testnode key/cert as a truststore
-        try (CloseableHttpClient client = getSSLClient(clientTruststorePath, "testnode")) {
+        try (CloseableHttpClient client = getSSLClient(Collections.singletonList(certPath))) {
             final Consumer<SSLContext> keyMaterialPreChecks = (context) -> {
                 try (MockWebServer server = new MockWebServer(context, false)) {
                     server.enqueue(new MockResponse().setResponseCode(200).setBody("body"));
@@ -190,6 +192,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
      * reloadable SSLContext used in the HTTPClient) and as a KeyStore for the MockWebServer
      */
     public void testReloadingTrustStore() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         Path tempDir = createTempDir();
         Path trustStorePath = tempDir.resolve("testnode.jks");
         Path updatedTruststorePath = tempDir.resolve("testnode_updated.jks");
@@ -240,19 +243,21 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
      */
     public void testReloadingPEMTrustConfig() throws Exception {
         Path tempDir = createTempDir();
-        Path clientCertPath = tempDir.resolve("testnode.crt");
-        Path keyStorePath = tempDir.resolve("testnode.jks");
-        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"), keyStorePath);
+        Path serverCertPath = tempDir.resolve("testnode.crt");
+        Path serverKeyPath = tempDir.resolve("testnode.pem");
+        Path updatedCert = tempDir.resolve("updated.crt");
         //Our keystore contains two Certificates it can present. One build from the RSA keypair and one build from the EC keypair. EC is
         // used since it keyManager presents the first one in alias alphabetical order (and testnode_ec comes before testnode_rsa)
-        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.crt"), clientCertPath);
+        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"), serverCertPath);
+        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"), serverKeyPath);
+        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt"), updatedCert);
         Settings settings = Settings.builder()
-            .putList("xpack.ssl.certificate_authorities", clientCertPath.toString())
-                .put("path.home", createTempDir())
-                .build();
+            .put("xpack.ssl.certificate_authorities", serverCertPath)
+            .put("path.home", createTempDir())
+            .build();
         Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
         // Create the MockWebServer once for both pre and post checks
-        try (MockWebServer server = getSslServer(keyStorePath, "testnode")) {
+        try (MockWebServer server = getSslServer(serverKeyPath, serverCertPath, "testnode")) {
             final Consumer<SSLContext> trustMaterialPreChecks = (context) -> {
                 try (CloseableHttpClient client = HttpClients.custom().setSSLContext(context).build()) {
                     privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close());
@@ -263,10 +268,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
 
             final Runnable modifier = () -> {
                 try {
-                    Path updatedCert = tempDir.resolve("updated.crt");
-                    Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt"),
-                        updatedCert, StandardCopyOption.REPLACE_EXISTING);
-                    atomicMoveIfPossible(updatedCert, clientCertPath);
+                    atomicMoveIfPossible(updatedCert, serverCertPath);
                 } catch (Exception e) {
                     throw new RuntimeException("failed to modify file", e);
                 }
@@ -277,7 +279,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
                 try (CloseableHttpClient client = HttpClients.custom().setSSLContext(updatedContext).build()) {
                     SSLHandshakeException sslException = expectThrows(SSLHandshakeException.class, () ->
                         privilegedConnect(() -> client.execute(new HttpGet("https://localhost:" + server.getPort())).close()));
-                    assertThat(sslException.getCause().getMessage(), containsString("PKIX path building failed"));
+                    assertThat(sslException.getCause().getMessage(), containsString("PKIX path validation failed"));
                 } catch (Exception e) {
                     throw new RuntimeException("Error closing CloseableHttpClient", e);
                 }
@@ -291,16 +293,17 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
      * that is being monitored
      */
     public void testReloadingKeyStoreException() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         Path tempDir = createTempDir();
         Path keystorePath = tempDir.resolve("testnode.jks");
         Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"), keystorePath);
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", keystorePath)
-                .setSecureSettings(secureSettings)
-                .put("path.home", createTempDir())
-                .build();
+            .put("xpack.ssl.keystore.path", keystorePath)
+            .setSecureSettings(secureSettings)
+            .put("path.home", createTempDir())
+            .build();
         Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.ssl");
@@ -336,12 +339,12 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.key", keyPath)
-                .put("xpack.ssl.certificate", certPath)
-                .putList("xpack.ssl.certificate_authorities", certPath.toString(), clientCertPath.toString())
-                .put("path.home", createTempDir())
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.ssl.key", keyPath)
+            .put("xpack.ssl.certificate", certPath)
+            .putList("xpack.ssl.certificate_authorities", certPath.toString(), clientCertPath.toString())
+            .put("path.home", createTempDir())
+            .setSecureSettings(secureSettings)
+            .build();
         Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.ssl");
@@ -373,10 +376,10 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.truststore.path", trustStorePath)
-                .put("path.home", createTempDir())
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.ssl.truststore.path", trustStorePath)
+            .put("path.home", createTempDir())
+            .setSecureSettings(secureSettings)
+            .build();
         Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.ssl");
@@ -482,6 +485,20 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
         return server;
     }
 
+    private static MockWebServer getSslServer(Path keyPath, Path certPath, String password) throws KeyStoreException, CertificateException,
+        NoSuchAlgorithmException, IOException, KeyManagementException, UnrecoverableKeyException {
+        KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+        keyStore.load(null, password.toCharArray());
+        keyStore.setKeyEntry("testnode_ec", PemUtils.readPrivateKey(keyPath, password::toCharArray), password.toCharArray(),
+            CertParsingUtils.readCertificates(Collections.singletonList(certPath)));
+        final SSLContext sslContext = new SSLContextBuilder().loadKeyMaterial(keyStore, password.toCharArray())
+            .build();
+        MockWebServer server = new MockWebServer(sslContext, false);
+        server.enqueue(new MockResponse().setResponseCode(200).setBody("body"));
+        server.start();
+        return server;
+    }
+
     private static CloseableHttpClient getSSLClient(Path trustStorePath, String trustStorePass) throws KeyStoreException,
         NoSuchAlgorithmException,
         KeyManagementException, IOException, CertificateException {
@@ -493,6 +510,23 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
         return HttpClients.custom().setSSLContext(sslContext).build();
     }
 
+    /**
+     * Creates a {@link CloseableHttpClient} that only trusts the given certificate(s)
+     *
+     * @param trustedCertificatePaths The certificates this client trusts
+     **/
+    private static CloseableHttpClient getSSLClient(List<Path> trustedCertificatePaths) throws KeyStoreException,
+        NoSuchAlgorithmException,
+        KeyManagementException, IOException, CertificateException {
+        KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
+        trustStore.load(null, null);
+        for (Certificate cert : CertParsingUtils.readCertificates(trustedCertificatePaths)) {
+            trustStore.setCertificateEntry(cert.toString(), cert);
+        }
+        final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(trustStore, null).build();
+        return HttpClients.custom().setSSLContext(sslContext).build();
+    }
+
     private static void privilegedConnect(CheckedRunnable<Exception> runnable) throws Exception {
         try {
             AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java
index df764bb3f47..048ad2e8e36 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java
@@ -72,6 +72,8 @@ public class SSLServiceTests extends ESTestCase {
     private Path testnodeStore;
     private String testnodeStoreType;
     private Path testclientStore;
+    private Path testnodeCert;
+    private Path testnodeKey;
     private Environment env;
 
     @Before
@@ -80,17 +82,20 @@ public class SSLServiceTests extends ESTestCase {
         if (randomBoolean()) {
             testnodeStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
             // The default is to use JKS. Randomly test with explicit and with the default value.
-            testnodeStoreType = randomBoolean() ? "jks" : null;
+            testnodeStoreType = "jks";
         } else {
             testnodeStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12");
             testnodeStoreType = randomBoolean() ? "PKCS12" : null;
         }
         logger.info("Using [{}] key/truststore [{}]", testnodeStoreType, testnodeStore);
+        testnodeCert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+        testnodeKey = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
         testclientStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks");
         env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
     }
 
     public void testThatCustomTruststoreCanBeSpecified() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         Path testClientStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks");
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
@@ -110,7 +115,7 @@ public class SSLServiceTests extends ESTestCase {
                 .setSecureSettings(secureCustomSettings)
                 .build();
 
-        SSLConfiguration configuration =  new SSLConfiguration(customTruststoreSettings, globalConfiguration(sslService));
+        SSLConfiguration configuration = new SSLConfiguration(customTruststoreSettings, globalConfiguration(sslService));
         SSLEngine sslEngineWithTruststore = sslService.createSSLEngine(configuration, null, -1);
         assertThat(sslEngineWithTruststore, is(not(nullValue())));
 
@@ -126,10 +131,10 @@ public class SSLServiceTests extends ESTestCase {
 
     public void testThatSslContextCachingWorks() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .put("xpack.ssl.keystore.type", testnodeStoreType)
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("xpack.ssl.key", testnodeKey)
                 .setSecureSettings(secureSettings)
                 .build();
         SSLService sslService = new SSLService(settings, env);
@@ -145,6 +150,7 @@ public class SSLServiceTests extends ESTestCase {
     }
 
     public void testThatKeyStoreAndKeyCanHaveDifferentPasswords() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         Path differentPasswordsStore =
                 getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-different-passwords.jks");
         MockSecureSettings secureSettings = new MockSecureSettings();
@@ -160,6 +166,7 @@ public class SSLServiceTests extends ESTestCase {
     }
 
     public void testIncorrectKeyPasswordThrowsException() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         Path differentPasswordsStore =
                 getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-different-passwords.jks");
         try {
@@ -180,12 +187,12 @@ public class SSLServiceTests extends ESTestCase {
 
     public void testThatSSLv3IsNotEnabled() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .put("xpack.ssl.keystore.type", testnodeStoreType)
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("xpack.ssl.key", testnodeKey)
+            .setSecureSettings(secureSettings)
+            .build();
         SSLService sslService = new SSLService(settings, env);
         SSLConfiguration configuration = globalConfiguration(sslService);
         SSLEngine engine = sslService.createSSLEngine(configuration, null, -1);
@@ -214,6 +221,7 @@ public class SSLServiceTests extends ESTestCase {
 
 
     public void testCreateWithKeystoreIsValidForServer() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, JKS keystores can't be used", inFipsJvm());
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
         Settings settings = Settings.builder()
@@ -227,6 +235,7 @@ public class SSLServiceTests extends ESTestCase {
     }
 
     public void testValidForServerWithFallback() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, JKS keystores can't be used", inFipsJvm());
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
         Settings settings = Settings.builder()
@@ -251,6 +260,7 @@ public class SSLServiceTests extends ESTestCase {
     }
 
     public void testGetVerificationMode() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, TrustAllConfig is not a SunJSSE TrustManagers", inFipsJvm());
         SSLService sslService = new SSLService(Settings.EMPTY, env);
         assertThat(globalConfiguration(sslService).verificationMode(), is(XPackSettings.VERIFICATION_MODE_DEFAULT));
 
@@ -273,7 +283,7 @@ public class SSLServiceTests extends ESTestCase {
         Settings settings = Settings.builder()
                 .put("xpack.ssl.client_authentication", "none")
                 .put("xpack.security.transport.ssl.client_authentication", "optional")
-                .put("transport.profiles.foo.port", "9400-9410")
+            .put("transport.profiles.foo.port", "9400-9410")
                 .build();
         sslService = new SSLService(settings, env);
         assertFalse(sslService.isSSLClientAuthEnabled(globalConfiguration(sslService)));
@@ -325,13 +335,12 @@ public class SSLServiceTests extends ESTestCase {
         ciphers.add("foo");
         ciphers.add("bar");
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .put("xpack.ssl.keystore.type", testnodeStoreType)
-                .setSecureSettings(secureSettings)
-                .putList("xpack.ssl.ciphers", ciphers.toArray(new String[ciphers.size()]))
-                .build();
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("xpack.ssl.key", testnodeKey)
+            .setSecureSettings(secureSettings)
+            .build();
         SSLService sslService = new SSLService(settings, env);
         SSLConfiguration configuration = globalConfiguration(sslService);
         SSLEngine engine = sslService.createSSLEngine(configuration, null, -1);
@@ -342,14 +351,14 @@ public class SSLServiceTests extends ESTestCase {
 
     public void testInvalidCiphersOnlyThrowsException() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
-
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .put("xpack.ssl.keystore.type", testnodeStoreType)
-                .setSecureSettings(secureSettings)
-                .putList("xpack.ssl.cipher_suites", new String[] { "foo", "bar" })
-                .build();
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("xpack.ssl.key", testnodeKey)
+            .putList("xpack.ssl.cipher_suites", new String[]{"foo", "bar"})
+            .setSecureSettings(secureSettings)
+            .build();
+
         IllegalArgumentException e =
                 expectThrows(IllegalArgumentException.class, () -> new SSLService(settings, env));
         assertThat(e.getMessage(), is("none of the ciphers [foo, bar] are supported by this JVM"));
@@ -357,12 +366,12 @@ public class SSLServiceTests extends ESTestCase {
 
     public void testThatSSLEngineHasCipherSuitesOrderSet() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .put("xpack.ssl.keystore.type", testnodeStoreType)
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("xpack.ssl.key", testnodeKey)
+            .setSecureSettings(secureSettings)
+            .build();
         SSLService sslService = new SSLService(settings, env);
         SSLConfiguration configuration = globalConfiguration(sslService);
         SSLEngine engine = sslService.createSSLEngine(configuration, null, -1);
@@ -372,12 +381,12 @@ public class SSLServiceTests extends ESTestCase {
 
     public void testThatSSLSocketFactoryHasProperCiphersAndProtocols() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .put("xpack.ssl.keystore.type", testnodeStoreType)
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("xpack.ssl.key", testnodeKey)
+            .setSecureSettings(secureSettings)
+            .build();
         SSLService sslService = new SSLService(settings, env);
         SSLConfiguration config = globalConfiguration(sslService);
         final SSLSocketFactory factory = sslService.sslSocketFactory(config);
@@ -397,12 +406,12 @@ public class SSLServiceTests extends ESTestCase {
 
     public void testThatSSLEngineHasProperCiphersAndProtocols() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .put("xpack.ssl.keystore.type", testnodeStoreType)
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("xpack.ssl.key", testnodeKey)
+            .setSecureSettings(secureSettings)
+            .build();
         SSLService sslService = new SSLService(settings, env);
         SSLConfiguration configuration = globalConfiguration(sslService);
         SSLEngine engine = sslService.createSSLEngine(configuration, null, -1);
@@ -462,8 +471,8 @@ public class SSLServiceTests extends ESTestCase {
         assertThat(trustManager.getAcceptedIssuers(), emptyArray());
     }
 
-
     public void testGetConfigurationByContextName() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, JKS keystores can't be used", inFipsJvm());
         final SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
         sslContext.init(null, null, null);
         final String[] cipherSuites = sslContext.getSupportedSSLParameters().getCipherSuites();
@@ -525,7 +534,8 @@ public class SSLServiceTests extends ESTestCase {
         assertThat(mon3Config, sameInstance(global));
     }
 
-    public void testReadCertificateInformation () throws Exception {
+    public void testReadCertificateInformation() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, JKS keystores can't be used", inFipsJvm());
         final Path jksPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
         final Path p12Path = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12");
         final Path pemPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/active-directory-ca.crt");
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfigTests.java
index 6dd9bb2b46e..511fd778113 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/StoreKeyConfigTests.java
@@ -22,10 +22,12 @@ import static org.hamcrest.Matchers.notNullValue;
 public class StoreKeyConfigTests extends ESTestCase {
 
     public void testCreateKeyManagerUsingJKS() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         tryReadPrivateKeyFromKeyStore("jks", ".jks");
     }
 
     public void testCreateKeyManagerUsingPKCS12() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         tryReadPrivateKeyFromKeyStore("PKCS12", ".p12");
     }
 
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt
new file mode 100644
index 00000000000..59ecbd22e8b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDoDCCAoigAwIBAgIUMVGoHuyNTjTFaoRmqFELz75jzDEwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMTgwMjE1MTc0OTExWhcNMjEwMjE0MTc0OTExWjARMQ8wDQYD
+VQQDEwZzYW1iYTQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtGBwa
+n+7JN2vweSUsYh4zPmh8RPIE+nEVjK1lx/rADUBY7UVjfTYC+MVKKiezZe7gYCNT
+7JNKazPpgVI9e3ZFKw/UxomLqRuuvn5bTh+1tMs3afY5+GGzi7oPmEbBO3ceg0Hi
+rNSTDa1rfroZnRYK8uIeSZacQnAW90plITI7rBBt9jq+W9albFbDybfDgNv+yS/C
+rzIsofm4rbFC3SMRYfrT6HvwDhjOmmYKZci5x7tsn0T+3tSiR44Bw5/DgiN5kX3m
+/kl9qg1eoYWbCUy1dKmQlb4Nb4uNcxrIugLB3zjBkfhMZ0OHoveKh/lJASTWik9k
+xQ9rEYbpsRbuXpsHAgMBAAGjgcwwgckwHQYDVR0OBBYEFJOLa7UXKtLPibgKeFh7
+Kq1+rS0/MG8GA1UdIwRoMGaAFGaNmN5mi9jaMW25MEWYgt+5OkDBoTikNjA0MTIw
+MAYDVQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBD
+QYIUdwsnIxjgSneHNVKT6JNCCsrQ3T0wLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/
+AAABhxAAAAAAAAAAAAAAAAAAAAABMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQAD
+ggEBAEHqT1WHkcF8DuOgyIBx7wKcUVQ5H1qYYlJ1xgMGrKFFZLUzouLcON7oadEu
+HLIJ4Z3AKD3bqWpcls5XJ9MTECGR48tou67x9cXqTV7jR3Rh0H/VGwzwhR85vbpu
+o8ielOPL8XAQOfnAFESJii5sfCU4ZwLg+3evmGZdKfhU6rqQtLimgG/Gm96vOJne
+y0a/TZTWrfAarithkOHHXSSAhEI5SdW5SlZAytF4AmYqFvafwxe1+NyFwfCRy0Xl
+H40WgVsq+z84psU+WyORb3THX5rgB4au9nuMXOqFKAtrJSI/uApncYraaqU28rqB
+gYd8XrtjhKOLw+6viqAKu8l7/cs=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.pem b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.pem
new file mode 100644
index 00000000000..63dca9651ed
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.pem
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-192-CBC,D097C279AD03E97C34B81B834539C0BB
+
+rQkO27X8phcP9ydaFArZy4SFDcoCGFsr2TcR0M6Vn8g0ZT9MIrnj2xHCmeMExlKT
+MViBQzmupBs0IW7y/ovpEUBgRd4TnSigKNF2XZ3+C6F4rkziTheWOlaUq7PIqOnW
+dTmf/WZDLlgms6hzrc7B447lO2FcNUDB6uXkxS1dAhh9T6DFcq9KuW7KJOWt9Dre
+75z6rh5nHboUw2i01VqJK+86aL5V3vNK3bjYcbIHKHrgM/+V+hFUNJ6aEepVf+G0
+jzicgEBJwtzOg4MTSqR2z3niNbz9LLBJkH+uR+tN941+JPFmEwsLtkq3OBH7nraG
+Srqz/5YcdA45jdZNme+KqorT8mNX/Hx2BsmQYnw+A0ZAqnSWeMcNZgFaAn8OcVxy
+d5VS0KFlQD7DPnZi89cyTRScKhcj451NhwOPWIE7w1UpNPVGRj5FBMAtDVaE2Wkc
+wuQ0nSwsC3EOvVDMe/fmv2VcoWceh1V9esA2H0n9NWQApGSqz17PPebwQadPX3Y0
+atrbbXT7fYTD3Ij38bmYZiDOluHiXxFchWxVUReq6nHJD3yo8ch2CKpx9XzjElLv
+6leUZhlIlq026QxGNVf+OQLTlLXjF8jGbRC31Y4yiHj1I12P0+dic0W8YvUkC5Pp
+w7NHMtYG6VHLGkPbCQxzTMS+JU24j/wMIokZWlmaRNDf2DZZAS0asQ/EOG/I1afc
+SXyHDQUhi0/8N7CJWE/i1xLrazFKiIkxucxY+WDqGrk5sZnP+cH0cM7Zja253NTy
+2B8LZJX360peCD15grkMt1oRtdMvZxF1rS/3BDPuANC59yg5i4eC+P39wkeMf8hu
+o8I2Hp5021mT9AWE1Dxf8gSZIJZxIeEgioRvoY1LOEfghe/OXQHR0SXJq9k5GNiA
+z4Tz3cfCT3WrHbReY585o2qtbpWu2OeSdxrv8p9kYi8GkevbR+nLq8NaH5WPRD4E
+b9RLUz1NfiK8DOW7tk8+gwsNun3hmil7xubw1uhc6a0OGKb9ijMS0rslYW9zeowu
+dkROuVwhUhXHOx0ZGWUGauJifERzICWR8Jx48/8DWD4xW3DkIRt5gh3CvzHcdSbv
+4VEFSyndWeyNk2Yc6+sX0H35Rngc7gjedCAn8hUBnUq9srfhGKaZ6bahlUt0xsmK
+Y3cxcd1or/2S2hONcN4NE2MfB/RRUVeHxdp34RPeW6L8qH/YZFxqt8dUm19QXr0c
+CdgSEmVyKOpCPebGJwPwdJEmbxPS/98AjiqOpt83JLUhMeUGhjawXvLzl0YEBJV9
++6waTV4Xl94aJszyvDeW/+cKIgeso9SeQSN6fLsXgdAVABCZ5yJ+liw6I84G0f2n
+D6e51P0JQAL8v28bBACdoB0Qxr9UTV/X8smGTwWobv/KW1BPdvWETsc7TrtWLZ6F
+qiZj7mI0h1oeyrC1h1+1oVuUTpy7JICCBloL4ir56lcSWmNZm1GRfar8UhXA7aNw
+klAkS6rYHH4gDxhvwd1k/pN1HlCtbmwLyYC/f11+89RnPr0FFW48qMqmwBls63dp
+4aAqneUiEow/evE4fBTLhFrgkvfZnjhd41IpzXfMWB5x9SHjrrS4/rjsHXcHUrAh
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.cert b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt
similarity index 100%
rename from x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.cert
rename to x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem
new file mode 100644
index 00000000000..b0f7a585d7f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-192-CBC,2F36F79E75ACA7803AF1BC1B70C2360C
+
+d4/f7dnpPW9DfhGXuGDx7r56BjQ64iNcsQdrC1CTZB363iAvBBeHaoJFaWpQOUmT
+WCBdM6YmRuMi6z4sMtm8Vre3i+Mu2Jti0LTDXqH8Y6FMpptxAYoFb9hkM0OvUGDJ
+ASLGTa1XKyCzZWOe2IGLtWIS8zkGv/fSXHqDJV8ddq8uLHGZXBxAye5oJUD0654T
+DasIllbiTlBEYlEJ10OyIAspS47n4LWVE5ggyGdiyIgh5s3lAMPO3+yenDU3/dzj
+YF2zHKnfe0Nx2lUQVJRYknQvFCx0WTGH9gNl40FovoOMHyRjEuOHD9PFNvL/TCh6
+9h0jJPWu/SsIiYaIqR0pDgqWdHXHA5Ea3s2+w0YHbv7DqkGXWZdceLUdZtcXCBJk
+P8QL9IWa7VE50SS6wV5uXX9tV5lHzMFsNGkwtGcR1UMU5cXYqckFXgoBqDN0fyWe
+V5iEknSJ4Na+MHv75rvRat0kv6upu9i5NSpYTc5jLHdWZWsYMZ/ZMiMoLBP+KAPT
+DQ3eyph/84BU3DePaQF3Rsp0ZvPxqQ361Zwc4zC5CKbHur1QX8WAY5XlBMfuBpkf
+CKr5wgwF+ZpS7zsfUpMPPe9Y1E8TWnhx/DtCVLEslBpr2u/rMaxPp6ev9/Wry7N+
+UFBOwodipBhlfSvLqjc511L+bXRzqXiINuW0eSKUQv0J/G0Ur894kJJ6feDYlskj
+JiZjOgOyyKhB+K9AXmkfRdvWUJeweL8pgDuYSyQdQ0zoUCZALEoYK2cBWzti/wep
+QPFD5oz8076aXNHKMHLsRmSFuEZb6IN0PtUNVf958EbrtABNIuoufKlKtJsEnUyK
+VHIEUxExEgyECiozKnxvhr7RQ9nTQXhNdgtec6jJblYnla/+OWAfHdxtHhBjp5AX
+WvLyUhmgrmLNdKd1KSzcXynBHgh0hi0HJXYx31FilwbxsdhwN1LwN/Do4T4qGkUr
+InrQC3ZHRuh0yAOPrwRFEWbW5m/PzVP/xYVgFtVWP7w38ftZbaBy5xPmtswn+PH+
+cIMt1Y9PaAlhLNpW/Vfn503T9M+05vu73vbU1xgu/B1kePOqE/WO0cOZl0KdaMmT
+wAQBKuI7qTACH+/8g3Uir1YSChLthH+1Gs6h686EP6ZydwXq9GYXXkNmJNJJsnmU
+RDjoT0F4XBKvcQdX3EeQYs3Af2yZWFDC59c1Ews2dqMK7vy2tYITbx2yn30DBDAl
+xvjp2izzmAgQJEG/RqCYsUHCCEv7wz3tpsSOkFem9IHZpR2h8Rqy88GH9qYOkgwo
++fKSmIgC4RLQXsHuh7RRuyNc2FaWDgRgSxs5V4f9xOSU/ZbUftYWnwEyCwbu3RJp
+CIXQFZhzU2t5l1Eh+x40rwpEJDXBEwmOIUO3x1oOqGZPPEQ674uMal5TRjvdOVGD
+h665Fpo5Xu9EQwQZHYddeRl/7yw8F6LCxBLgHlngKRHHGDUHlTscLfYRqNh+x3jT
+3S8dfaGzlnwdQEx32gyLAV0/nsFnzh1AknFMT8jesIYF7PLiAi67PNyNwRCc7TFp
+jpKvzkDRVP72bivTmCyP5aKR0Q2oIrAw51MMinT6R2VaoR7COjoVbqYsRLwkxu+p
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt
index 75aeecd0c3b..5500e5db4c5 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt
@@ -1,23 +1,34 @@
 -----BEGIN CERTIFICATE-----
-MIID0zCCArugAwIBAgIJAPqdcmPxQKraMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV
+MIIF0zCCA7ugAwIBAgIJAJZQBp49qNzmMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV
 BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEgMB4GA1UEAxMXRWxhc3Rp
-Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTgwNDMwMTUzODM1WhcNMjIwNDI5MTUzODM1
+Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTgwNzEyMTUyMjU0WhcNNDMwMzAzMTUyMjU0
 WjBIMQwwCgYDVQQKEwNvcmcxFjAUBgNVBAsTDWVsYXN0aWNzZWFyY2gxIDAeBgNV
-BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIIBIjANBgkqhkiG9w0BAQEFAAOC
-AQ8AMIIBCgKCAQEA0wNzfQ9K9WIs9Le6pfiEHdCI97zGZRqOREKb+zFoDllXEEWW
-Y5mfXRlTYXbxBiCIV5DDW0yaXlleq62j7/O/6prNUBiYo5sK4Wfj+1UlXLmMikkv
-bShm9XzBAXHK20coLJTKOH8QOnFyuVYUvHPCLsoEhQtCHU4zoGaaPmk8w1JU/bfR
-+kWmU+x0Ak4rGuRWvMMqg/bu/W/1LmESO5Jsm2CnIyB/22vB08kFn1pO0IIrPQhr
-dXqPxddzxc7DuAyyMyzsrLi5msugynEwm1CTBNL9cG45ujNhWzd1rqQe1HF94mEw
-RinFe2Ui+vLFpNbh8EesLsy0p18J3QkGQ/0xjQIDAQABo4G/MIG8MAkGA1UdEwQC
-MAAwHQYDVR0OBBYEFLUR8gs3uCNdLIwJlkp7SwngpjfhMIGPBgNVHREEgYcwgYSC
-CWxvY2FsaG9zdIIVbG9jYWxob3N0LmxvY2FsZG9tYWluggpsb2NhbGhvc3Q0ghds
-b2NhbGhvc3Q0LmxvY2FsZG9tYWluNIIKbG9jYWxob3N0NoIXbG9jYWxob3N0Ni5s
-b2NhbGRvbWFpbjaHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQEL
-BQADggEBAB73kmak89jgW1q3wIUMfxisokCsk7XuYqPBpc8D06UZBaArCvW2cuLP
-5PLI2bR7fSIhgZmlbawa3adOLZ9n9lOJrWUHTh43kKyiSX8EobE0T/MSGVMfTEFu
-c92erLS2gSDk4wLahzm5htd1h1KwzJ5j5kdzlLGaQsnxDDjbu9BiX/otEyCl8n69
-AZKOXoYscl2NxBgr2V6C2frthJFfQ9Gxzg5q6V/P3aIUyV0xsC3Ve1wdfXqNeRfO
-nrnFbKRDsxJAJ/JtO3GTtqBrFjods0sepKNxFg13r/QLJnYjYW6t7o91JZj2AFOs
-1INZnCOAMV3vR/FOwwOT86HDgrwSy2o=
+BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIICIjANBgkqhkiG9w0BAQEFAAOC
+Ag8AMIICCgKCAgEAqLiqAPYBBEBvSJCiQOzV/NPgr0kLQkZGaxQ29PPoKbsKXVp+
+0Uyv9DUitPw+G04h/eOC2wiu2u5sn2rD4Ew09L41qiaViQRY6dHazAkPVq8Chet/
+GWrc+umLJUKLU7MTyC8ozmKjgkyvIuvQ6ndEP31z3JG/j9DsBAX8NWIIJSm9Jaz5
+XUS4fIXwSce141k8zb39+oStyA1qIhv6n59+oNIuuXu1XIJzjQnZCnyAO8/9i7LI
+uoL93zu7xNT+ns6Tt7zhweUQEI8FeRdj/k/Of8prbaulFH9oM1g/lnGKLV7E5vh/
+foP1+SRW+MWRjAUA8MExTCtvFhRAb3x6FYzCPX3VERKn8M3m6Rewz/LQ7XG2VzdM
+/dw/JSZilAWBZItkY9H1InTeIz9Sw9Z53um7tO5nzq1QQxZijh6n9vzSLoFn6qA5
+SDQl2YycjBE35i/9JBUl0KmVMOfFzpoWLiKzTJMRzNJIeRxJl3MvscbRl8fY0Kco
+XQ+w84QMTo+Tn+8Ztfme4uGtHHCTRzrLSo+Hyh8BTm9jJKCaUbMnNW1p2LoxJlq5
+ah+W8QRLaFbWt4xl9TQR0kVnoOyfSGx+3SmTBODa+6Wg038eim8Tw8K+EBvxynWU
+sKF1ReL0aOZLi2zqPiQTUtR2y5ue/xjsFWx+tMMGTz3Ci6UoFs9yKqQ++qkCAwEA
+AaOBvzCBvDAJBgNVHRMEAjAAMB0GA1UdDgQWBBQLFB6mVsZpiHNMkxOgriN53HZv
+ODCBjwYDVR0RBIGHMIGEgglsb2NhbGhvc3SCFWxvY2FsaG9zdC5sb2NhbGRvbWFp
+boIKbG9jYWxob3N0NIIXbG9jYWxob3N0NC5sb2NhbGRvbWFpbjSCCmxvY2FsaG9z
+dDaCF2xvY2FsaG9zdDYubG9jYWxkb21haW42hwR/AAABhxAAAAAAAAAAAAAAAAAA
+AAABMA0GCSqGSIb3DQEBCwUAA4ICAQAQtteI+//qUZ7pNQ5tBuwc24UlSH4dbHFr
+79pA0YN/Y7Y/PfU60YeGpPf/QzZWLnRk/8mELWy2Ok7KzhlNjr/nAksiF97LUUnl
+6dP15a4ATzheYAtv/47In8ShOrPZ7YG+VAmNWRB8Rj62RuE5khcoL8BpWzCHL8Gx
+Kif7lN5SOV06ExGHrBr4Y20QtZsTgkWhAP1UC5EwXxKcchCAIrb60Rx1OShzQrxz
+I0RF4mfJ06Jad6qUePkPoUm1lPcFfNvAnJ+hBA210J87NSMFTSsgXT197upnCdhl
+9QmKHyk12oVoMtTtf2kc1yTZQ1fnm/V4/PZ7ypyhy+jMsIQSTwEKQr7JYEQoYpdt
+yTMHc9L4gPkLTwAbxUx/M1lSuQ4yD3SclBt77CxCM8I9Z76qCVakalb4VhplRbsU
+sYs3dgvslFOAJKoOIPeTqm+2L6psxiV9WxA6tvEAk6P/AwjOK6Y4hclnoQDgNI2Q
+U+T+ZPi5WRmibs0WxfQv4Vv4MQG/LRT3pwsKWQ76TzgtHKNHgtAKd9Sn0Nylacsh
+yhDbvI0jnvwoOlUPUVWerSJLxzazG5mRLi94ZxZMb+7TUUtDBEPtkCquCAA2b4Kh
+ykHbmBqhC8Pn9qc9iP0uHuuQETUl/zd7E2ZD3RIj4uYYspovihE5D1Svi5m+3NuS
+sCsfHRjmMw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.jks b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.jks
index 5a5596dae48da6686f4fee825d3c76cdaf0c8b2a..0df3fdba60daf866548fb5f6eb9be1de8ae09659 100644
GIT binary patch
delta 3666
zcmV-Y4z2N>5~ClG7=L7s8Wr6D011dNf(e8$4h9M<1_1;CDgqG!0R;dAf(dZsxuawg
zT;MDy9y7HOX{8g`m{ODd*LxA?dybEj!?yR!Gd6aI>u_UXnJOOZ;Et+(XKrmUfY`~2
z^U|<!NHzk4MBUH*1o;BNB(QP8+sdRJFS)i3FX%6^T|5f&_<u+!tn_evdj@6?LI;qR
zpdq4kXHspdxZ1K<R)SKSrm_>#G5l(HJA$8OABuzcxy>*jTp41PQ$q^EF!zR1%2z3}
z2malaxy|9Fuv1F;;nr!jvFV_(W_ipY>sablBHx_Q6AKM+6z>{y`{hKpMtF8DC-gJO
zYN$2{C1S{ThJR&dZ68*PGaFF)xsC_b?qi#~nrpyOM}USTW@gzTs7fJA{r@PRJDIT2
z9%6%+U0+8By{3Kp8Y+nZ^^I{~!q{Tk64Y5R=;`RY-JKmnU%-!-WK&NK%F^XpK<{}-
zI|8K6AI!2mq2GJ5;J2j`>kbfbL*1Nn00LgYvf`aoP=7Ed+nOt3!sm!)MqEq1%V0y9
z=z*Jd;0Rew74tgx`An@9b~fF!-D6ijllDb;byrW>{l=g+q8g4tX9YD4QPB!L;~B7R
z3v{0V*Z)|Juh)#`CO@Wo_T09%QaP_?XT&?iG{Lpx%G3aoHjW>$pQg^e_@e|-IBt(B
zwfbrew|^B4q@5EX4*z@d!ART}cbPiYHQ;{2oa#8+<Z~MW=-Yt8eU{m1Gk&e{R$}`Z
z<N5%sigfY<OMqw>5uzuI{oE`L%SIE38%!X0H~-*H>tA3)c^07K{gc1oBwTWWuX%rt
z(S_}a<r3aSY42WL?QHe+M`ZM;<2v5l#d;brZGW!2ZWnLp5l+*3Kk11fAupN7y5T1?
z0luLP`13_A*Il(h5MM64uI2PCK-ac!R@mAWtAP#1g|%A*-avHsqE@WLYqmpwqVqv#
z1fP!Ub??B;pNz0f^5WdC(-R9vQ>r<{6bG~+VzYnsy+e0$GNYIQ&<y|04Jx_%|J!#O
zbAPpD;r%N6sB3_mbm%ZO;Tpp3$Ow!8$MoGNjv?&T75Fiqb%v$dvy<<g40s#vfa<f%
zY8!p&59PI`E^49v{6JeND?*P`EQ#@B%P^#t^6kIJt&k;cVH$wEuiI%`u>&_r9AkFs
zvc$ost=mEggr-2lz#iIZp@_dB!2WqNV}B0C5^C~p-L9I4C)=|S&i;bG!sU|cK_ZQR
zB#6lu$gl3aJ)O}=0B}0ZT3UT&Ds#2acKRO%f#VIeD>aE@M)8MbIyk<2+N4CBLBV-o
zGvqZaAPtcdT_7^ncY^;|89BmD8)UAQCa|vv$EO`Ly?WOg98^)yH0AGh(EtvcuYb3y
ztQHArF9H@{j)x&cCKeD*6sXV}w}<cOCAx%IfSnk?x%#Q}jRI9zdqr{e7i}@H|E_or
z=`KbmH3)W%pm%LP-LCU~rp3N1)YS|69Nw#AYST`6Lg&fu+2VuWgIgH@ORZCmv{Ak2
znmlc4Ess$a7XVW2pcx5%w=?W%+kd<91yw*E(_`+N#DkPpIlC;YgF7Awekf;kHc@)w
z)@~0|T0?jA%jSFN4jd~o?;khWM5AZd0Qg%p<!KM|DD}gOP@Y+T2_J=3*;R?L<qnZ@
zpHN{o9t8c7%BVFjGZ{J*ON3_lbj~|g$rnrSjhpmmpvTx?g2aR9T5OrDKYvS?Tgu@#
z8$Q6q`r%jP<ry6HrYN#eG2(-?bq5X`LVB(WQg!&@NTsr8)2`XZ;6wJknT#r<!d@0^
z9~dIME&sq>J`#^Vi^Rq`I$v_}*zMuH4IpoT*B1Y$rN9?|2b3Mfeyx_*Ks#eK1(enM
zOo(H~$mp=Y!FnEuw0Cglet)lx)4%?3kg^`ReTs(1dPfC0n$QkQi7K0$40T;uL_jtP
zoL&c{t)-{N5h2bwY&g6wkZeUEjK7bOtO><8fTl%UT<TP}sHgp3;iq&tf1!xvb95c=
z6W&v}iZ^BLr*`?b_jyr60Q1%5#yAVc?$P^Ak2~<RaSnnFFdQ8*WPeaa=MnPz_Kc73
z*ljXC8Um3#gVFjg$5A#=0~bZ$d%)7EpY42#+Us*V!WdiH!}It>HTn|N)C8cq!Hg_k
zYZE;s87kPvm7S!28UO|D$3Rsl&H6K%`GSsB-LQ&fC0<<+M(ESAz~OEzvld|en`v7a
zmSAx}i!fgacnQb=e}4iiPdyD?E$-V3ax$`IJ_$ne_1Rddj(~k_pu}f`+flOYP#i=n
zIWc3vi|Ku`IOE{$yOyCtiThph(-5|zrG>lTC_NlMkJi|m=!wK*=U3vFdse>M6l8)T
z+x-_E4~^^BYiH2|zu%mxW|?XS)}%aKZcT8K-CJjpP)ERF4u5IzZXeq4Bwja`GN;K4
z_W-e*SE#jZak_UoOfB6JEG{R%MXoraZOY})tAq!_UwQ<P;WC2oz{9=oEY-o;)QBEl
z3NMh-C09aFGaXPtc9n^8jkG0o(Ygdq`85=mbnV6CrMO+66SLXDO)u2B#+Jo0d4PJ~
z<M!cm2$$vbjeqOx!O2J8*vLBsU|)MW(@3Ed9S2ZeT|4nw00}s~RGjr0qSLRSPObm>
zTkYT7%QPB6(aen}+|#NTl%_31+&K8&EgY_@LtM|mA5<~6zvcAY84Tzti_80005~})
zFL?La-5fFC|5{i<orWfPUy+&(kHz9OtSCv^{5+6mXnzOCQ$0c`9>CfvVf3t6ib8l}
zmQ=VVe8`cL=?ZxBLWb`#2fR(~5>=d}g0c-_0A{ZSTufH(<TjiCZb=}J@8xf{-Wp^w
z+R*5{sy!%|$)-y}j(|On8*Yl$`KB>%c<yRM`xwIvtv<Lb8s}e?IWJ1r)?3b{6v}8f
z@oC-DXMa)oAD6@Z+i!v3l$2~N)AD3Oxu+J>26YAh_$@9;bp;r)@&0EPLNv9~kKk!<
z`n{_RuEn$b_rwH9S?&`u07acEk@T)iF2)UTXqrZeVdY|w*1H!2gsum@30$*qN5?eG
zn_$%^<*K2#cikKI;rdxp9F`9rm16Ft_F{#5Mt_~DdoS7`=1$0t(Fw>drhdAO#+^D8
zDARnVyMQ0572y~G@oAd#{)fBs_=XwJ{Ean>4Z`jaO-Sc%fp%11e?*_8L}s*^suSXl
zX5tS<5(g1w7GdJaPBd!HFG7$N?vW1DP(d&J(SqUDtqo@a$m%_7O_trfkyR$aZ_~A9
zsY`bmh%d|On3u?6Y&q|Parc9$w06M?1zZU4Jk2dP#U3&n{4xLl00966SS~d%IRF3!
z*D!(w(=dVqyPyLC0RjRE0G3b&o;|4C=95YVU>P?tGBGtWGBq?>7Y#HsFf%YSF*PzW
zH8fh2Sp`dff&wBi4F(A+hDe6@4FLfG1potr0uKN%f&vNxf&u{msJN;C_5lPyZ%B}$
zK<w50)8Ma33qnRlYZNy0^XMtN3SC-$(M+%OH6pb9J{wLU{o{h$2(H@hY@cew;7m01
zzBQ^Qm5Bsc>CxKE2@h7U0)^{;8EV}6>5C;oi&L|I6UZ+p&SIm2Os^vA(CT+YKYerD
zk-v}7>;wh;HDU-QDZM4E`CUY~e1-5y$F<{GJk7oR`h=~>4Qe7A`k$YEpwce6d$nAG
zbBzhv3VeV&&;5(C$hw04-#fd+)c&5%lefI#!Q~JTj|F)bWByOhf68lZtECiwXfs$p
zmT`)IEndXt_<w$b_4y=L_{EWo1px5DF-$9O78F2le0mj(!aaS}5fZ2H&F1MBu+Q?)
z?QynOH%$H9KP4t&lm&rgi)3Tb^&)iMBR^8Z)_LCPyR`0S&aF^G7GjPbr}q5PE`ew2
zpgBl1CE1Lej1f2HFa0DHCD5srFz3b2niej9BC|}B5zNv^c^pZXb1$*R(U-^A(5EO}
z53uuu3{H=epYIv9`JUq8tsHQZM>@+&kB7=10ZwmYB%qp6vnMreY1q0kCR({_AC~b1
zOK4Wsw~S@=G!fE8XQ1q#NNj%HDU$@?+WV!T(|;a{Zxh49z7QMn%5{{mp>;*#^l0WJ
zON(skJ|q)T)OO38p8pu^6>NUA!v;@1!i%LS7SD1jq(1tolP(7#6blp{rdGyjh;vMn
z6QHgmdE9nyIFn-sqkjSc01&p<i2MKQQJ(2F4s8bP9NUE@NPZn`acl3|K+%JLW41p%
z^*YgqhNSoZLpD|}bY%a@ge`2gI!?;Y8BLDA=K@P27v9TJN#*I&_2sSrPdHv+3vd69
z$e+ZaI<wjBfxc7;jad+UMn1Mi;W?5QC@;WiTQG+&!LceQ`+t<)QaN2Z6A_238u(*v
z5Ve~Vf<>VK{ZtE)Fkcd!au9$bw))W=bvY<=LcDV$L`C9f$<v}*cdC?l`46E<wUqY-
zeA~dBpP>W|wa}kCH6sO0D<EAxeeUXK3D{-z35p*nHQI$}GSuyVX&l!i*+W<7oAr47
z_It{lq08vZuz!RSPXP)-zR6%jC}Njw$ukFY()fV+3r_$W#Y}%QSyH(UG7of|lpA~O
zEJ8EFJ!ihE309<PR`^yLWktIbv5Pl$3+$9rfF!C8Aor81Z?-S0Y{n&hTM#<7@c@&f
z{{sllE2cPw$!DPe;53bpQ{?_+__<jbqHfI=#q=-XTYva50lzI2_ooXASq}P7I4vBb
zM}p7_chslQ+@)#DA<7Wjyp1EC{3tqA4^dT~ts+auHnSU<kuEQIXBJFv?$c3ALj*(Z
zkSeYS05)%ep~^wqm>QuA!{_#=J&64-9_x@1H6{NydlP0u-4Y|><`}Y?FNzU44^*#<
knZDfHk{hrq9~~IxGnDbwQWQ%u!*BHQX<gk$;L(G)yTg9@@Bjb+

delta 1983
zcmV;w2SE6vADt4A7=L3ciyA!u00jatf&~6B4h9M<1_1;CDgqG!0R;dAf&}W?r?<Ff
zqfy7OG-`{D(8y-pwjU?5S!#^1Yv&~G5SO4w?Lh!Z%*wdZ(XFc~-WO0NJSQ$blB|H0
z*T1Kcvy1}XSwE`u_R)j&kkE$@D-Otw!k5t;M4Pte!_Fnc$bY4RsIP}TpxF|ZK5Az<
zQxp2YewhvcbJUsjZ!f@0<q0sJ#4?ovD%|$v7xCgInwsH9i%dLY*{g{h&qEGFv`kde
z3JBv3et+2<S}2q#;v1!rIHo3=ZqlA!(*V2ijCApt%Byi;rOW)mo$-k>GMx<yPaBCb
z2~O%3&dnp<mVc0Wq0nxARRMtTs4%#C=|~3);u<IHL*3kXtHW8U<9jFe+P?;tlBV%$
zl7_uGyb5p;aB)M{BFk$x+zzZM?(SxcqLsAT!pG((7ofOYy3ewiJh-0f!YAs1`L7^S
zyXY#khT%LXSoOv&$r;?dp3PO-M9p_azy)(GEq`2;5r1J{sm(pg9khu_{Y3NZ;mswT
ziIkc2SLZ>m(5Wer1$FAwC{R>yf*C`5JzgPUBD~`nWv0t-)Y?Wi2!^(I#vOeMy;98d
z727@B@6v*%VB&Wh5jRfuY`19Iy^TTA1KRgMWbNrD`t+{u1njo{a#&`fav-TeV^l1#
zX3|hGS$|~G(yVB>LQ3OkEDMxk)Ju+yrq@dnQ9V~>a-q~t6JlR)lFLn2o+)mcTNPGg
z(?ws(l&sM(d)1tM@Lx|AJ^?)>8jkn}A#UgA-WsO=4N+sm&hKhn?i%`ZT1q{LoYv5X
zX8Cj<moVH|=Cj&-<WbG(Y9*Q^k!7AVUP<D?wtu7RPfvdjlIPVp&9VI6G;dG2GIXjo
zG~z>>l+o>G;cG+1ooK0wA0c%_v@kzt9|tee`dSL773#+Xy>-&qo?g1z*|0%r@{1lp
zUfSRy%AmLR{gNd{PRlq5<v0$U58GHvV=?$ju57c{Tj?Kc2wmzDPtAjPjgIH8t~yQ1
zt$&i34tQbrZfy}?rI2i^_Z(=Js|@|W)V;&nq4e$5=Cj?o*EIF^r#%J>e!yk<w=GS&
zLD*7zJeL^qQ#ScmjsHI!iWE*v0tk^X4dwk-wJa*Svh3~QZISWn<tcv6%PvSlz3G<g
zsBROt8C91Fn2S~Fdp;^W049XKFZ>O|oPQ19u-AK{)#Uxw?h9J>3y_qcW|*0HvViD_
zaJW3^H)4(~-q;!h@-EY~WgoJ9k;GUSm(^45$~DvF{snY?7{8kxMc&auYNB2+j7}WU
z2a8O=qWII&fELe)3<l>t<$uXHjRj1-&@=a~^;1N;hjR<}de)`Z9GE5ozfXwezJGiJ
z!V@>8T1l4&lCe7+<sL9fl{YLQ7#qudGG<41>;*ZyE5JqZmcb^x1!+#y><aBT9t9*U
zlL(Y$!uDH5_l7#@#003f$3k3i_U)NRy3sxr9L5A#+*^@%+I_G5gVkK>#3J*a<+va`
z(`&#{J&AuwN8E65W91Xa;!!h$lYhLtzO4LQZuHJ%*0cTu<MY*g5iV<blZEzsh}_3v
z_9KX{p~C5N>-%HTR)VXH396we1ys>_zWI;+jY1Hn0coz%aBPI!(WX8Li#ec|c@TGZ
z5vb_^ZBBk&g1vG1U^Ni{9M;gCabawlKlWQLGx2`7xbNObWA-?~?9rkQk350VjUMfX
zx{n51PVx))F~e=&DjApn000311z0XMFgXAK1J^Ku1Jf{q0=u9C0s#U72>|+?a%1s8
zs@jv#31AsCGcYkVGdMFfS{Ds6GB7kUIWaXeI5Ra`livwTe}VxbFbxI?Duzgg_YDC7
z0R;d9f&mWzFoFRJ0)hbn0Mi3=eGf|YVl4Exx~2Gp9ngsPyvAi3jzmJ6`!Q$^SyvE6
zmSdTpT^UnhcJT%vh*yxqTTGf>SzfEHqwn*->YB|^7?`7*3gKtt`&A`exr~ZQFKsAh
z^?bnramw3Ae<&=J$~b=zI&pHjRusH*!Y;}Lg$qI*PBWlpnm%bf!%|fJx6%4Vrc>;6
z0!}L$<W{`HDued!{crUyVG=u%Y@1-GBOrg<Yr)gW1)o|@(1I&H2y1nEkHyz>#m>XH
z46-vU?5w!Cn#-WdaWI=ulLXTJaBexeGhtgdb*`iyBGhqx;$bjGDaCtbBKq>hq}Ji^
zN31T)w5MMQ-3bOm{V|P`v<x8>wGr|QH@G8REQ|@3N_$HQ;HEd>lj#hje*pmi9`}-F
zr1RL|TUxikg$#cftfD}yleO+*qrs)m1Jk7$1)wVm^|o^2&*bvR*|d9oA|Zj9rERPl
z-KS11pJ(<{iLGS^P98UqtfEPO1fj7sPxBHPQy)x0Zgbt9thBa)Amrl$+J`y0hSzn6
zQn1XPW9CP5l(CvZ$?*(0f7`pzVqf|#6CkDXe!T&bj$Vc=a$SwY7%SOcg4z1*gpprE
z(Xq}BYUy9k-J%r9T`{mN)q7kWeR_>~7tWr!#cZTQvl2ij@@+eDleVC17CIfXOCF@7
zaTX1Cuk;Hhb|YbKt?rL?C7AXAQ>@g3S)3z)F<tLR@lL}7lk=g&8G^hL%WC3cJtPx&
Ri=fP##=@bc$$^?ZK~8wn#-IQI

diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.p12 b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.p12
index b2efbc498999d110986d2280d2753d35c107d649..592bceaaf0aa6f6d0f9e9271653fb9bffd2cac99 100644
GIT binary patch
delta 4274
zcmV;j5KZsI6wDz%FoF=s0s#Xsf)I`d2`Yw2hW8Bt2LYgh5Pt-M5PL9!5O*+w29E{_
zDuzgg_YDCD2B3llfG~mvd;$RgFoFh=KOTR9$4}2FjpAqm0tf(rf(A$;S^8TGogsz}
zZ@H*M3QT<E1C9SZikZjJbof$|@md$46$EDsAAkU!^VYrC(>;6$;*9Do?=k4RF&WGd
zaFOhAJt9X68VocpIAAqvCZz}Py_P!~frq@5l-YsI!A*(e4bO<FH~@5u-6b6Q=L>&>
zzC*GP9QIY9>VaPcoS|_msv$M})wFTtXQBR2Nr=K7>e`d$!^xP2Bu=|6cF~=M7l_eg
zuY#)9EfFH?#myV>6Z%kBD*s3|?V7!Ja56|iPO;U{iF!K;&{m5Fi!F9LDrggi^tA<=
z#Zy+o2uULNO{yWS=`!&Yl%kiKe20H~YZ1Zm?oyQx)-hM(Bq&XMm--W#aR;s(tTt)b
zrimFOPvrNgYy=cy6dzg3p<5Yx9)gB!ld!D7R#5$=)lNQtUb|0rLJCuD;Z$hPMM}ga
zdwjQgL9(*1Ig70{Di!FC4gAa@f{T_dR^^?rrW;BHf{Uj?bs$lDa(C*b6Dxm{_je-j
z6YP7mTl6AEf9(n`WImQ*sgoM;CVuW~cgU?qsPm(P-vJq?!&!OLStbey#niEU*~^4%
zIk!M#Dx0d4H=#pc((HKgE{48p3f}^(n7N+KT>5n9TP*DI1J5K<ANeK!00))Jb>l)?
zJsG6oRA`8I@ne9Lw?s$>+_QfrED&R8L6!r$Rk<}$K^l3p6Pd=(JRn<MR3^Icy--||
zY{ri?EgIf;=?epcuyR0cZW25I7Xbdo0~w{(z@y4Rx(bIE!y3e^b=LC@lrRt`ZK!%J
z#7NOIjZV!Ruo(Ea5}!xKaEq*~>%_8WmUCv!Bbp0!7w|hPGOJrf2Vj5PuL<AtKPxoM
zF9o>h&q6?YjiSf)N0$3$=-err52f1^pSDDAK!wOJ^F%7UMlsxs9|+RwAe@_RDeN7`
zh|^^=;yF)?4j%`xh(3%~k6|yut&oAqU8j8syJ8$tD6dM(OJt-V7jQZlHqHVCufBEB
zAHk@-ZPNctNn$W<#{z%+{hPny{ET%Xv^S;Y`<iE(y~RM065*h5x>nFk&`ijWE@o}l
z>r{nd-7iv`x4$N!d4JfR0ru)_u0m8aUvhNfkWrrRT=qSOVvZ*Rqa*yv4S6(LrNFsN
zf@H+aFO{=w97nHM=gb{}a8ykz%kLSb!NpP0-i<f99YAt%Pw9V)SO~dE8nUn-Z2v&C
zoDqpnKKHa1cPE}_mY7O-=%Dcl3FE%R^pWWXT9B0pBP`Q?&<lU%SJRdV(&tp8Gf7nH
z$!yvnS_kSywe8;W_}otXm&tF4%^m%4l4W;#OJ3F?GH$=r<)vaO*m<^)to8~BWp(4W
zY0LQGQa33I1do4iqR&D<xiJEGrOV9;Mcp7A7(tT6%3fOfyA3T<y4j$t&fu8zC3NXi
zY-a7X=OX5>H~$#yLtg0C@uwcg*N`-*94ayG_9+(e-WCZt#d{nN0%k2_Z#tCT(6|E7
z=F&Byhjdzj>UTyl&AWOG4JYlxX1T`@%A7jtp^kZAN<4oT-?T9djtiyoef!bQxyb5y
z@P11!U2Y0;GqD*=wG9JOrOSXhDe>atgv04tG})xCzCH+#RC*T`*7ypymOY)^q7n`t
zhOTp<*3bip7y0Yf!-rpeVeuc7ngeQIpW2x$!d5BSrAz<gNidlj36T$U8OK$xOz1H*
zg1iA*@V0-cGx_PaLKW!Z;5%|Cq^wIn6ADenr!if3P~nuSFkTJ2HUTq^5B2C7VPGil
zo}#h)+0FJkJ;V<F(5iep-<zIVnsPG7UTYw4w`imIUwx7?iHj6Yt%&xLXK23**VEaS
zH~Zjt)BNfZ_U^5cxK@bH_6{rR&vmK`3Xm+bq3wTKxcq3UBF{g;+_w^&=YqR#7=^(9
zI^J9wZo5|oI82XaQhD+<S%IGO>Uqy7(u#~p2uB$KxfB^bcnEASq3-EV{$$8_SC$=V
zMpUUo`rEM5+*6I?bW<Jq92JvK`MhY(U^{#ztTH{5U93Ek5V$Vi>8BeG|C3!sF&is7
znD~DVJS*g37aIetC*t93C`1rj_WsV^lqzqNF%K)*m9<#Ld$x~=RU0>f7vx0<&@8mt
z1VJpw3yVwFdgK{B)lcc_6+b(_ZdB{yFbHcFC-54ISHgD?o|tpdVxTT55_KvEF;ajq
zf(hUT2`Yw2hW8Bt2LYgh3DE?C3C%Eq3CVvjf(gY23o3?4hW8Bt3<?1Ppn?f@FoFqk
zFdPO7Duzgg_YDCI0Ru1&1PCPyhL6V($Aba_2ml0v2~aAnWk*k)gzkszu<sz~gpe0y
zNdOWgG(6>)al{q;l|(MG*llr~lY}!J^hk4E@j%(?B~h=^!z4BFi^ap}|B1BdE`opj
zy}W7Qnr2HuFii;aXTwC6vy73GKkQEhYEAT<vNIv9xL%l;^f_Jz^6YA$Wyv^O*L7+Y
zh@_X=hpn5wv&apCqtoW%WJ9}M%F2Q&3r5lo`feOc_63=z+?O&t<wjc&6VYwc4YE?(
zg&Nozba9AZq1b_1q0(PEuNmA1M)!ZFRL41T^;HQ3_(`~Zk0>r@X%^r$N>+^$8m*?H
zN%a*TRc#^=4^dxBp*h9<^zCFfWE+s*7IoN3WkiU2`Tv~fmlx3Ou&aXZ8C{1?n6m0&
zEGZ#g0f>7#r{eI?)g%-C+;x90jG~~@M8aCFa={~HIM%Or6zO{MD=F+idoO>KBKM8(
z)8W37vDxqZ5hDPE-fJ1$inQmy>H!S>1c32z?T=zFa`Fm9kz(sZ$lAzWno?<{h<aRB
z3zsX?O$os1Zd{HmCqAw#=dlm6$|jWXzo5pagPee9>Z^FEV6vcAy~$b`rd*=*;`);y
zQr@S63=Qh!J86KzIoqkBg&=<hQu}^!P03KqR&)qQo%Xo{5(2fxSH$+6<^gYyq0Ar`
zKPL0b#&zq9TFM7$K&AWfC`YixxtT>e1uGZ4a@@Digt!7)`bsj*1-n)73f2UwvQ(pg
zx)tulxh}es>isucSg+ZLgY3QknFY6mWAXmCvqW)N`oYyE&L=~HwT6Goo=cCrOi;k%
z+ECD84K`RTNhSa2Ab)Ybr2;p_9sRQ&xv{;jWE6TNCw4AepD6QPkCLKC9?ouO4;c#O
z)sjC)M#{=8MX0s^h6a5TV^tyx-xp5Tg!ySAZfq7A2uhpUS7d~f+;%GT;XihA=s;jy
ztJA`RY&3)3g0tOeo|=D;WPu3~{{e#J&~1ABYd;#nEjj+C;U{K}QGpy3X!Lcf9S=0Z
zlh?zjE|bk&h$)n1luZ5!pO{gDlsGAyAw1HQyljz^OHbVw+$n0A8I>t&3F54TO{E1Z
zBs<A)TxI4~JXvg5hUB<-z^l3htS}i$bhUg=CP2SftW13C=a7H#P+Ew)>=fhtPS2H`
z;<22am^%kA56<|&l&Qs=g5wj#+d{DXzLk+@;kE!KPN8~KRtNt03mHML+$2QX;?2Lv
zS*vy@CSac?RYf6VtCCc4Dg<V@fu7Xv=rx-Ka(_e#mj05vQSV8|W7{wvh1mJ7Z^@2p
zg@5iRrrGv0bA*2%;+mxvRasN_-ZvM~ow*^_@{*DqOq;PuVeSmIW-~rmiM50YVO@1M
zFFWIktTa%mH5A=I_@fhr%~QAY;^falb|dWe;fNON^^=Rdq#$T}a}><E4cOCBy{Uk?
z0x;w$UvTcJH032%Z(d)9`+)&CXtj5VlBnkY-K@}d;|YHoG8}hpE$AqG_aZBN*U!@f
zkUwC%F%}SX`K~X{npaIb974GZV7$Am3Gzs+PVUC;)Y8n^=|ENuvd6lzcD8=M;(2%>
zt9@!OHkN5Xn2qR4RlqTEco84x-JNs8nK?lSbRR(BSRkHA?uOX4doz6B#6O<TWkOuW
zZ8lxVZ1I2Kd|x({6P?CLw<lz$xW`qE>QDx~&D3c9s~`GO6&S#ZD(X+?0g!`GJVP?V
z{=FsoCgnHA7RYH&mX<MNFBa7mAyRj0H-wK}F@UZq>+@dSpP~zAX-xxZdJJVC4;y*B
z`?=QO2E)$xq+_q}9tEv)Ft_866r%*SL*A5O-Uxr^MR-bxavh`@s}X2#qNA#m>Sg6}
zk?`RFVuba&+FlP4If!xV9OykP<9-cGuu3U+dJ&sg>bLCI`?loZi%x2IpCDa>5|?2i
z3Z0}+{xHdoPu~dWGNj7Q8$dNF(mF#4AmN1W7KL5>E>IyL1=@DTVctHiF*oucAdQog
zT8V%1hJ*(r<6?ZlVdc%}=ifR1Q#LSjF6&M+^05Pn^<@crp{Y=4;cVhE>FV3paBXJ#
z#|qn&!kT`Hw^xxGItq*w2VxNtj%{H6CUu@@mDWf*FG`TmiZ9<fNN^WEF$8yp=yS(#
zOR;z00jzN7N`lvu<u)R)YlhudVBlsq%SM0wr(!Dw5lh@<H8+k@4$yl`&`mv!L5MqG
z{_v)!HvxH0u5wfI<bTDHNh+B`GmiAR%*IGua{`PtW4@KCg?%Mf9o5>SYSKu7YY(Nf
zc*J5uCG8DA1Rk2H@0n)-_n3m+dcmcRV{7sh)m)kL<OfhbYlqE9L-zR05a_+7pR#}R
zj!RCx!r?uy%DbqHA}Z+9Yut^kaY7oz``kFcY{M6>>_3|eXax529MT{HJ<_XFtat*G
zr9l1fy=FZU!GL}#Jl@ZF^YJkq%I_zqj+Iz2ZKqmacBc|jV|Io7#Ip*mO>LVVdE}$a
zvPn=?8G>k$IlSTOvnq6Hd<LYdU_pN@*9dZGyJ}@<oy`HPuxg>R+0Gr3c-2}x-yV<$
zpv~ldu(RC;4~TwuD~(Mwn&Mb@6p~3W+>YZ8+|Ttu4is%x|HlLHNWtO8?mNn6g+DqD
zxN+$3BlfI}BZMu3-R!n{Oi$HawlMr}cX?_foI7}}g&K}A^n9C?9j?TpW;uU{iY}Bl
zE=3$Fkr=qK@5lJ$wMJp#y$2F{^wq6maq+O+CZNsmQv;^>LTF<id$1X&9K(9`lLDC$
z0K`63ng8&8a$7ODF<cvj1HijLi|G<hw{J!jsq~3(QD;!^S<9o})_;%&QGf6SJwxa`
zytu(A?Im1jA-cTtWj~`?&KZBHf!vz?eUuOP&L?b!oa=b7c;Cc_1fLd<p0v9v<6my6
z1P)<Mo-JW}ltM#1yA55MVhC2_j63gPAb#WTL`J>2jw}QCD`j+Z^O6W)!`^pKc%J_2
zd8`u!AEzaFoSy<hEAQ<2SGaG1z3MJf_^69P^o;&+d2x;o$sO>G&=`MnXOVZ+?S-X5
zdq(G!buUn+ix2+6II$SrC&HrIAcfaxc#gT#PNyWH4P;>M55)r>sJ1I02+)Y76R{@e
z^hb0%Rt?G3*#0%+;88cVWcO`0QAB@f_KRO`qIbYs)vGHp*eU$eT9xF5SjQJ$R!tWT
ze_?zb0`&~sk#CD09|j(dH_?~Ax<j*q%}SN#ptw*xaJjDUxr{mg4-1nKy_3@kDn~dS
zkjtS5l;?<dUjUE;+iLBv7921!Fd;Ar1_dh)0|FWa00b0ZhqeSYTK7cI&pP2@+P3@L
U<tFR|2r*&EkbGnl`2qq600uuAumAu6

delta 2589
zcmV+&3gY$5A;c6vFoFs|0s#Xsf(ix(2`Yw2hW8Bt2LYgh3HJno3G*<53GXn11dj#@
zDuzgg_YDCD2B3ljfG~mtd;$RgFoFb;KOTSFLR@$+mmm%T0tf(rf&@rS$(J=?SLygw
zYo&{Vs>^g&F<hxA#C1YVkFLPq&P3MJ)z<L1ERo~K!`H#z<riW_J>~*q(?1heY8m6L
zoj3pW?8QZWK=_)jALDC_8E1gl_Zm+v_=#X#8mFY5$Nq;N?3;!#&f&HyWriAZm9>A^
zu4X=MvQqHXji(DoH-wAE?;-Kq5*i^ldLTsVH$YB``qnzKsH4FuNuZ-Cc!o-Sm+dWO
zwf1Tghpxy^dupjE`EO|56QzW#E6xmGEd%kNF9iTXOWdJUE>!G5lnIY!X910o`N8aC
zbImUmW%cA?MqfYMAJxb)bK++d&sKlBuMF%0!8HFcGbD?V)!Q*Ho8VBO`d=9#)Su5P
zh29%wDdXo*A9w-d^a+|)idko?9V;xz#A<aeln|mqPK%*-V3kb?pF8<k)68|Guk<`8
zvu^brW6tyG1D+1p?0)|*(DphAjDx)a0eiMXy%hg8IzcM|hCEbm0#`C!`Q3jwv&5BI
zq#AkM<OZsFXB(}e4ZV>*s(myNvTr@*J}lB0a0a08RjQwT%Yv5nZVS2k<H48@j0sMC
zLlNSCno8tf)B^eE5MQ_{SNT^3Z?B$ZJ5i!v5|ItYS5}XKkc#Twu#yo_-NeZwPoQ7o
zW7c?)`iJ4m>Z1N0w#P2>8PI>KD^jvx(43ZHf#07W_O#UsF2;}?oe<9EVcDe)ntXY1
z>$KWxRNRN`M7b!lhWeICsJ8k@OT2HYLU0(m#_flwHB8NEu3hw}@W)h8s#Qhy)`uOa
z`!ts4&s&?%VPj!UJA45O4EV!%Py{)Jd0dpthdDBbZZ?1fvFgFqz?FYwvyET_5~@lK
zqVaNgILK7FQnnVjxh6x`7nua!<mwj@k)uz^9c;lzx|+aq?u*H6$+hgy2<NHTT&_D4
z(CIHn6EWB&bcFZJ;n`Y+I~6rcu*s+#s7oXulSJ~cLZSC>WmW!xtT{66?7UVO9gS+i
z3*wdRN~YYqCfOW&0mFYa8sr|+?E5W3WxFhumUAZx8E{(RUaes6%uCHmqk_6e++{I^
zi6@SAhMd&mb48(sAhj;5nn4L4k{_FIv)?cvJqyHv5r!xm6}aoNO+}IGDAs-)5eKwI
zz)Y9Bdl6%?ltaLp!))(_M-VVOxINC2UPISoQgXk3=@t@`ktu)UvCT)XW_0Egt7A(d
zxVDRp0_)5nl!W7xc422~?Yf+gtQ9Kek5fjba2^lLV@;qqg0mCd0WG{NO13!|n~%q+
zpYKoPV@&srkY(!OUEKx17G>*wvj1$W6~~@QB%3>yOz|R|<aDsN<=3f1{#r~7K5NCv
z5Sk@EB#Lu{r@4PR5Ignj??MQPeGyaaY(#tGa7h=5t6;Qt2+`B9zn1KlLz1tP0Z606
zVRuE7sr4-Ik~aJJ@?QB@W5Hi;ixu4bTS0VJ+1LYP+V}^J5PFNjInZEBGr16Q%Vl?+
zxhrTef(2Lx2`Yw2hW8Bt2LYgh1xW;g1w}A|1wk-^1wDTT3o3?4hW8Bt3<?1Ppn?SM
zFoFc?FdPO7Duzgg_YDCI0Ru1&1PEMvN!?_|+Z_S|2ml0v1jw&FU!)>$zZu>l`Sfj#
zSIX(=Qq}lby~%#Mv9N6Wi<ZhHP!}zM#fxCTmdP<%QArB!Ri(AMoyQ-voUWC6ehb_z
zhg?S(s|J6Bc9Z9BQ=+nxT_X$3nnW5ieTR$ncn_8{O(!rC`PjO9#ndW`#;Z5ojF{(~
zgUmQaEYRjNb($f@W92K>X!f#}zMd@sRTf)MsCe$80o>*3FZYP|6~)8#CWORO38Uw?
zr^QofK{_H!-YKAU?!CGze~6W@TS3uAitUrW(UyNWXK>T-k+`a5NXmm81Z2<NWqtmW
zSiiD=6p}E%BQkRYT)?ak%3ej*<mE4(iSp40Zk)r`0zAX;0eOO<PsiDvZ_%F|8@_*T
z8?iJ6cdo$ykS2L<@(Y9uR6>G;){uyUT&M(n3th(Qka=XDxk&pYxf%Z%3I+`eJ`w(5
zp!9z|*lQJwj`zqH&^}<z7y=pT^=_TBBhyin)Ta^}719pVhcwMv6K|nyOSYuQB<{GV
zo31OB@h7hB;l6kq^c##*)9ii+y*vdxZ{}NUg?KOYl&~LypJ?aIJb(#`%?bq^W`ZK(
zmnSP}ucj!><HsEH35~iwdNEH-KzEs;@Ya8RP}12ji=6;}{7gw*S*$|8SvqxJ+0pK0
ztQw4^ug_)lAzbRc;m&g)u?_E@XN7UCM><OZj=#y3RgTeZtQVG3^DvrUrU{j~j+Q%4
z2I>7VSIXi=if(fTZXSt*veIIzLE|;p7~OAJ2qvmL<&?JaPMWn$c1zq?QNTY*qE>$~
z4qf{FUndBaZ4~SrP<n94gu-~~T03gEOQ>LEEMhAB&qbLG%kM??jxuhX74+NiRmXx&
zqi7b}b#cC4VvIU8Z~?FL;`r_f2j5GmpMaHlVM`q~n&Cj%S(q0>cyzKWwn&XM(g6;V
zF54IyDa`q1dSV{!u^Z`D&G>aW5rBW9V$erF!nqwBP({Kt22^$gumsu9(SIu#8`i8`
zxT)u5UHvNi=-ZH`_)nO|BU@rgUfyM_A#j+-020pe4$$|-?zjEm`(X~`RoHj|Oe@D+
z65^ZJoxmESb6jG3?^uUsgbhi4qAg9wrv_k#7+uI@`&73inHHx<k^#k`$Ju{(eMgQp
zfk~Z7$y6dQY#T|gz(JLa2s*|WfJ?B|&J|ry(*_uIiU$IVYmKrgZ^XjVDoA4T_KncX
zE2eNLix)~PQY+xz?@{|)$rX7&ZQe_M@eHUpT1B@r{cxw{OlPNk#`CB@6$^Q84rT+n
zQ5wz><*y%ZGV+1Bcv_w(oGpK*Kt~6(w!47Nv%$@`iBOYZrk}Zvh$Xkak?XC87v@OB
zE9PtE`(%%4QkC8KfnjAK0WfLQ4=(>G6wV1>;Tdm;40eBx!zwvjlM^}Mb{E2_*$vp7
zd8l{m!O5XLr7?O`X!{-oV75NWP!f8bj&XY1WMRfa+U{Kz<hEO4fgXQazkG@TTh{Yj
zXCZKZzea-O#G9{_XyE_#zQWZ}_V9VP3bWUC<Y06XZe-~|N(FD~FOH=z0$#m|anScg
z<wFHmJIbT&=s){3Q)-pRY-fl|I++?~!s;y1Hxnz(Su{7E$H9vZyv|3yJ%7J`ZPmY%
zr)F!6o#fxXK!QT3nEwegIrFwU7jAY$lUoofM^w_R`ngS=?EUi{4T8V*6+Zd+GB7bP
zAutIB1uG5%0vZJX1QayZuxW7B(R-~0B&O}1Tj18(TH6E&ZW@xP{v=Ct0s;sCMm^y}

diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem
index 131f25a36e1..f8251130891 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem
@@ -1,27 +1,54 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEA0wNzfQ9K9WIs9Le6pfiEHdCI97zGZRqOREKb+zFoDllXEEWW
-Y5mfXRlTYXbxBiCIV5DDW0yaXlleq62j7/O/6prNUBiYo5sK4Wfj+1UlXLmMikkv
-bShm9XzBAXHK20coLJTKOH8QOnFyuVYUvHPCLsoEhQtCHU4zoGaaPmk8w1JU/bfR
-+kWmU+x0Ak4rGuRWvMMqg/bu/W/1LmESO5Jsm2CnIyB/22vB08kFn1pO0IIrPQhr
-dXqPxddzxc7DuAyyMyzsrLi5msugynEwm1CTBNL9cG45ujNhWzd1rqQe1HF94mEw
-RinFe2Ui+vLFpNbh8EesLsy0p18J3QkGQ/0xjQIDAQABAoIBAG4zFTMws2LnPLco
-5I8c9FJlv8IxkiLm9Xlsl+1dZgxoV/Glhd7KMdH7EfKKnWEnyEyCy6N4o/22Cpz5
-9+2ujIHHCoZT8hMvO0Vwbj2pv/Nm66Cki+IkAXEfD+TjOp+9z5tIsTtEEIOVD+b+
-r6GjirvIwQeCU6reVlPkgZUqfJM8Drh2vOKzTqalJ7Ud4lsHeOaCvAaLp5RKz6sl
-BSX1m3UyRkrOQRFNeD8FwXGgPmxZCbCdLka8em9b2IHFZmfFm+R63NeOWx4wQIF8
-Wh2ncLf3enQU9Xsti1bQ3Q1SALG5Xjf0rtjlk5LklnQ17MuFKLgsIbJ5e9Lg82Bd
-DyrwuMECgYEA+orKql23N9x6J2GdpUHIcJ465b8o8DK/HPE880pwkMKzxaKSMhuP
-VmOi9lJl0QzxL+cqIKoDxQxr9rzY8/fRa+FHPn2ypJeV7osyDCZKcwJyh8PfHD69
-XIhBXlcr9qLLXj1EqKEMGnCAcbDTiLPsReYfpJ4elrtE9gNstIqqOrkCgYEA15w3
-5Y4aqmjCABQW3SyVah772xYJWmscwxJWTrMuBm4EUoMjx2M2qHf8xLPoay4Q9YTR
-v+HtaTw1LAC7tTvuPIGB3Iopj4pyhBWX4m0kxicV7clJFB3eDrl4Dy/70IEJtWxk
-xxdEjxwgFhChWOeB4IdcLc/QnGqIZq9v1WGps3UCgYEA2jU9dAoxRAbToS8EtWTI
-5UT6eRT329Pm3ZdCBYdE6N+a4TvJZrjYK16WFUEIin1iJt2HtHE+3UUHVxt9PWcO
-T44ilcu2ZmHb5Kn5Cdnjp61vAM3YqOn/1PKbNEZp9OIui8KC5Pd26iQnyl0+r2Y9
-F1fpIV/RtmgeFMJA0DGhN4kCgYB5jvzAsmZ8TwWZb7oYRmuS4hcSxRDO5nbX0D3S
-rxiyzVIohwEkB96oCsX6zIer7dPn6FMtQJ5OZ4B+54bsvfYqLMXfZhHaeK7poxb3
-tJs4EI8/ee12pwQr/wseXOo2+plTrOIAkTC5Ep1Qc13UNum1gftJZNHzbeirJBwE
-DMTVhQKBgQCN7jA7Y7eyXdm6fQpCn/gOkKb4dGhKPlrNOrRoD2S/L1YBJjreGZPm
-NARLPO0H1Wrkg54KSIvf69COjVJeyw8L0HGE2eQQOwS7dAvy8rdj+UvABXgaQCxv
-wJsTOZEycAeCyp4wJOLeksMB58bzIXPDOPNlN6buWBVLjm33q7g3Rg==
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,50C44E2D186DDB06
+
+rofPrFk6MlImSnj1Gx2qwUl9N45Ao4u6FoqxMvw2F3X/MqnZ/0hvA+7Yh65C13Hb
+uTMBfSXDMpaWd3tV+1CA6fEVfMCq1Z894Chqvw6ygHzIMlCwDvar4jUEYOjOsRQE
+9wzngqGSd+uVz93im6NSFvt2LJ77qKHE8R/REX5c7fr2eq4rgZU/58q2RxvXXnWV
+rQGnAPoRIeyRStMKaV5gytIxME+88XDx5jr86a4wr4wraw7mbRlJG1ERWqNfSCNP
+tzBPo5LXZMQIvMs3qieclUWYH4vL3gKBcOVmKlsUB/0ov3vC/nrCE3BmvgsL9SFj
+ZUCrGsVH+nsRj6mzHSJtUQVSXHvoRGVh8erBCt5H8KmMyv8F8aiUERWeOX63Erk9
+2x4g9trewa6CCXXB+sKKCwqP06HNajjPo5WnTEPPag6WE06O3//yduagcR1QD3Os
+Xw0pyZQxf2/4nX4jLNipHCyuYkgcDCarRG4/AgTTI5+cv3CS7C73HwVqz8v5Dh5P
+I0ZCdqeK7R5wSlF58HzVGfv9lyjvUohSiQrCDQBhjVrnTjXR0kc0lE0GaoMnOsPR
+X0UAIxRnpM43q9A/7z+VEEV4CtzWAt6dIwqFaHr8iVHROy4UUcoC7aMa8rVq4PX7
+YX+vBInOiY+APK/mF22Ao3Ho7uehDTIeBfbF1ED6ugTJvhCIiObsCTh79SDEH1PC
+zvVVWl3SO8Std5KnuoQMEqi3kv2I3mBlGjWshjVz856OudX57KTcEAkrJhUD4SFb
+cwwDdxY0I/jTD0/oE3VL/d1qhh8mt6oj74z8a68BL7B7xo9mgPuVO5z0ZblizulN
+6S+vyb5q0DJIpPDkI0IEx1eEiNiM+WkTHG28SCZR/sq3TGH0B5o/qQzp9O2JlwLB
+KiZ8qAbzsYJo0VAIAYBuWdM+pKXcrIIoqa278Af4nmwoSkBzyx3wIGZLWYrzdGd5
+bx+S8OXdDp7YQraFvtJmNC5KqVY8wsZr77oXyQua5iRANh/5AWKbQWdawkOj34KT
+DFDsuLIYCrpInor2JoSMBagq1PSh8FzlzvY+Tsof3U1LpP4cVQYtgeSu+0TPhqFP
+L8PYaqeLfPGB3Yt+hPBMtFA21jgv+BSGRSqwQYfcpvaO5+Dy8OAKRlzwc9rRZY4I
+XBU56WtRvJHUaGPFiAizgR2w0//SPRtVXNIyfS3GjkjnYofx8agXVKhFyL31fCUv
+qVvJ3L1MPynp0voOLObiMIskU5vQLY7UWRzb/wbGknfurAV8xQhnJgcL4+VeoCBO
+JPjd8yz7FWwdEXhMyOB8GZNzIGDxY3hsfDR1uNwx45264e2tjcW/ZPUoC1lTO/v5
+U6eYk9VZybvJaeb/NNDQLUCwV3T4TEFSvDSgld/GFZNvGBqZYwuAhBKbkrO1f66l
+6q4fZgG07uBobOsuvY7Adr1Y4iz09Yjw2F2G5SJ+liZ4rijeN0+fkvV0vwp7GRiP
+2WmQIYvdWSek4EDJaEQyUJxLTskveyHLRDV2bo+QhSuh4bB6pCzNoS8Kco6mr1Tg
+hx47UrsW/w/KKx90Yl/NUjCMoX77QeHLKcDNreJ6xqGNfQDmcTt962uHm68ZwwFe
+qnWxqCQ9stXNh7M0AuG47hAOPEBZzpMRN6ek8akOuZKo3YzdYVneWofav53OwMMU
+N1Aj2ktX/IIvYiYbhF9RfiwYJ/UfVOjaiBtfAWgLdY7ZxQWQWL2Ld1Zf4SJ0orAM
+ffy3rFAn8A0HpFZOhpOi/IciwOcGvWtJOmLIIxEpnpYY6333poo9oiN3+FYKhUlL
+E94Mc7CLnI8Z5TlMdqdE9EFQLUQRjeUxMWwqd8o2s2BNI6vtgIwWTJvnOOQ2YuaZ
+hFzGrIef+6Nz2Za2jORj7KyfKDNEcc9YXugBFefXUmU8TkstDpM3BNPk9qjBX4//
+D3Nz/8hFugujw/KPb+hFy2knB9jZrmqgDykj8Pf8KA3iWWc/hEr1uv/QveJcJzjp
+OSOx7fYjJ9giZFDcbcc3oFFseo1IVzfUFGc5rb3y0KqZd7XS8eKPiw5X9NTZIYaM
+ynos8jsYEH65zDdilAMpqlA0Daa+2Az7661Gk7mMIsNyt1eanyYbVJswqOu2IzFd
+CjmaDQCeTqAoxOzLHoYxusM3vX3CkQg/e1ZNjqbCNB/RtPOj83rWbf8+Iujkzah7
+80v0B3wwXXlZQCk6Wq8nM1vK2W9k37GwPCalibMD+McvOrgblWwUKn9snWLSXym0
+d+QPjobJzH8DezMWbE+LgaFB7O2F4cC18WRPGZLUa+Wc9bQJODbxrpfvFE8NcR7r
+PjqXHmcSxbIAJdPzcclKSuibSzhh6QwcZrckoMSH6gWVWJ0lfz/UQZAeCU0MkboR
+hpOT24urjQ3yr37ScsuZPfPQhaYr9ML/AWJ9bhotoucCcs7KGrGOc4uXo4thuZG7
+cBO6ZBDHIiEx3njGQVpCxNOij5gmNYetAQARkmBVJiaU2i7gCR6sQB8EkHOjHrGW
+5iNbXsNOKGXQzRFe7tA2DOSV7Lsy0Gbr6/xf3NA4oDzKU06gQmT46CJ3wHWTuZ1P
+pUn5mJNBJFcqLP+XPfQ/bSzqu81TYCm2/ZTmJw9POgdejYglpeGgHzKuUZef4bJA
+8l9pEa2suE89+Ag4WyPb9izox3qObVUahDuulgBDXLkOJxBpr3oN0JKFGUxjvG5Y
+A5DEq1Ya7i64cNmAc4S52cHmnBsEzfDZscv4yL5Cp3XbBTIbE5G0bi/0FxfJQ2gU
+T3TtNxVkFxtvoSEl2yNt5aq1hZZlNOVla65w8Rcmv7IHZC4GT4PaOuB3IBrr0Kc0
+BZFJEj8rRuNiegvdnXNcbMNH7nXgnQYhZTZH8wNI35VntByOmrjc4GXgpdK144aj
+1KwQ/klmJuvmJR+3FI0BauIFJvbe6kcsUlCvI4E5QApVMMUUwt3bH4aek+7MjqPS
+1dB0L0a95MV4jQUaRVIAwXeQinkefIj1wYpIKnCL34Lc5EF/QVd8xNF2uL4JsLxD
+ptjilmER7nuBvtgJd3uLmGHpzB8oKMVf5CK1bsWqGPUTffpUM2fz0/gMa1/aOt1l
+q2OEDs6k5OScHHYSHs10ObRyD6yehNmFCIVSSgz2gu9h9D8xHshKR7OMmFxFkAlf
 -----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
index df1456c3790..7d329781fad 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
@@ -42,6 +42,7 @@ import java.util.function.Consumer;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
 import static org.apache.lucene.util.LuceneTestCase.createTempFile;
+import static org.elasticsearch.test.ESTestCase.inFipsJvm;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile;
@@ -103,7 +104,12 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
         this.subfolderPrefix = scope.name();
         this.sslEnabled = sslEnabled;
         this.hostnameVerificationEnabled = randomBoolean();
-        this.usePEM = randomBoolean();
+        // Use PEM instead of JKS stores so that we can run these in a FIPS 140 JVM
+        if (inFipsJvm()) {
+            this.usePEM = true;
+        } else {
+            this.usePEM = randomBoolean();
+        }
     }
 
     Path nodePath(final int nodeOrdinal) {
@@ -137,7 +143,7 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
                 .put("xpack.security.authc.realms.file.type", FileRealmSettings.TYPE)
                 .put("xpack.security.authc.realms.file.order", 0)
                 .put("xpack.security.authc.realms.index.type", NativeRealmSettings.TYPE)
-            .put("xpack.security.authc.realms.index.order", "1");
+                .put("xpack.security.authc.realms.index.order", "1");
         addNodeSSLSettings(builder);
         return builder.build();
     }
@@ -288,6 +294,23 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
         }
     }
 
+    /**
+     * Returns the SSL related configuration settings given the location of a key and certificate and the location
+     * of the PEM certificates to be trusted
+     *
+     * @param builder
+     * @param keyPath             The path to the Private key to be used for SSL
+     * @param password            The password with which the private key is protected
+     * @param certificatePath     The path to the PEM formatted Certificate encapsulating the public key that corresponds
+     *                            to the Private Key specified in {@code keyPath}. Will be presented to incoming
+     *                            SSL connections.
+     * @param trustedCertificates A list of PEM formatted certificates that will be trusted.
+     */
+    public static void addSSLSettingsForPEMFiles(Settings.Builder builder, String keyPath, String password,
+                                                 String certificatePath, List<String> trustedCertificates) {
+        addSSLSettingsForPEMFiles(builder, "", keyPath, password, certificatePath, trustedCertificates, true, true, true);
+    }
+
     private static void addSSLSettingsForPEMFiles(Settings.Builder builder, String prefix, String keyPath, String password,
                                                   String certificatePath, List<String> trustedCertificates, boolean sslEnabled,
                                                   boolean hostnameVerificationEnabled, boolean transportClient) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java
index 6447be7e69c..1886dd4249b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java
@@ -60,13 +60,16 @@ public class SettingsFilterTests extends ESTestCase {
         // pki filtering
         configureUnfilteredSetting("xpack.security.authc.realms.pki1.type", "pki");
         configureUnfilteredSetting("xpack.security.authc.realms.pki1.order", "0");
-        configureFilteredSetting("xpack.security.authc.realms.pki1.truststore.path",
+        if (inFipsJvm() == false) {
+            configureFilteredSetting("xpack.security.authc.realms.pki1.truststore.path",
                 getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks").toString());
+            configureFilteredSetting("xpack.ssl.keystore.path",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks").toString());
+        }
         configureSecureSetting("xpack.security.authc.realms.pki1.truststore.secure_password", "truststore-testnode-only");
         configureFilteredSetting("xpack.security.authc.realms.pki1.truststore.algorithm", "SunX509");
 
-        configureFilteredSetting("xpack.ssl.keystore.path",
-                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks").toString());
+
         configureFilteredSetting("xpack.ssl.cipher_suites",
                 Strings.arrayToCommaDelimitedString(XPackSettings.DEFAULT_CIPHERS.toArray()));
         configureFilteredSetting("xpack.ssl.supported_protocols", randomFrom("TLSv1", "TLSv1.1", "TLSv1.2"));
@@ -78,8 +81,10 @@ public class SettingsFilterTests extends ESTestCase {
 
         // client profile
         configureUnfilteredSetting("transport.profiles.client.port", "9500-9600");
-        configureFilteredSetting("transport.profiles.client.xpack.security.ssl.keystore.path",
+        if (inFipsJvm() == false) {
+            configureFilteredSetting("transport.profiles.client.xpack.security.ssl.keystore.path",
                 getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks").toString());
+        }
         configureFilteredSetting("transport.profiles.client.xpack.security.ssl.cipher_suites",
                 Strings.arrayToCommaDelimitedString(XPackSettings.DEFAULT_CIPHERS.toArray()));
         configureFilteredSetting("transport.profiles.client.xpack.security.ssl.supported_protocols",
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java
index f9b1be65736..6966b7edf67 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java
@@ -100,16 +100,18 @@ public class PkiRealmBootstrapCheckTests extends ESTestCase {
     public void testBootstrapCheckWithClosedSecuredSetting() throws Exception {
         final boolean expectFail = randomBoolean();
         final MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.security.http.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.security.http.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.security.authc.realms.test_pki.type", PkiRealmSettings.TYPE)
-                .put("xpack.security.http.ssl.enabled", true)
-                .put("xpack.security.http.ssl.client_authentication", expectFail ? "none" : "optional")
-                .put("xpack.security.http.ssl.keystore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"))
-                .put("path.home", createTempDir())
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.security.authc.realms.test_pki.type", PkiRealmSettings.TYPE)
+            .put("xpack.security.http.ssl.enabled", true)
+            .put("xpack.security.http.ssl.client_authentication", expectFail ? "none" : "optional")
+            .put("xpack.security.http.ssl.key",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+            .put("xpack.security.http.ssl.certificate",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .put("path.home", createTempDir())
+            .setSecureSettings(secureSettings)
+            .build();
 
         Environment env = TestEnvironment.newEnvironment(settings);
         final PkiRealmBootstrapCheck check = new PkiRealmBootstrapCheck(new SSLService(settings, env));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java
index 14b0a58419a..212ee7ea499 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java
@@ -20,6 +20,7 @@ import org.junit.BeforeClass;
 
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -92,8 +93,12 @@ public class ESNativeMigrateToolTests extends NativeRealmIntegTestCase {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", home)
                 .put("path.conf", conf.toString());
-        SecuritySettingsSource.addSSLSettingsForStore(builder,
-            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks", "testnode");
+        SecuritySettingsSource.addSSLSettingsForPEMFiles(
+            builder,
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem",
+            "testnode",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         Settings settings = builder.build();
         logger.error("--> retrieving users using URL: {}, home: {}", url, home);
 
@@ -134,8 +139,11 @@ public class ESNativeMigrateToolTests extends NativeRealmIntegTestCase {
         String url = getHttpURL();
         ESNativeRealmMigrateTool.MigrateUserOrRoles muor = new ESNativeRealmMigrateTool.MigrateUserOrRoles();
         Settings.Builder builder = Settings.builder().put("path.home", home);
-        SecuritySettingsSource.addSSLSettingsForStore(builder,
-                "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks", "testclient");
+        SecuritySettingsSource.addSSLSettingsForPEMFiles(builder,
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem",
+            "testclient",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         Settings settings = builder.build();
         logger.error("--> retrieving roles using URL: {}, home: {}", url, home);
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClientTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClientTests.java
index 0d689adcdf5..9b8c3878a03 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClientTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClientTests.java
@@ -49,20 +49,23 @@ public class CommandLineHttpClientTests extends ESTestCase {
     }
 
     public void testCommandLineHttpClientCanExecuteAndReturnCorrectResultUsingSSLSettings() throws Exception {
-        Path resource = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.jks");
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
         MockSecureSettings secureSettings = new MockSecureSettings();
         Settings settings;
         if (randomBoolean()) {
             // with http ssl settings
             secureSettings.setString("xpack.security.http.ssl.truststore.secure_password", "testnode");
-            settings = Settings.builder().put("xpack.security.http.ssl.truststore.path", resource.toString())
+            settings = Settings.builder().put("xpack.security.http.ssl.certificate_authorities", certPath.toString())
                     .put("xpack.security.http.ssl.verification_mode", VerificationMode.CERTIFICATE).setSecureSettings(secureSettings)
                     .build();
         } else {
             // with global settings
             secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
-            settings = Settings.builder().put("xpack.ssl.truststore.path", resource.toString())
-                    .put("xpack.ssl.verification_mode", VerificationMode.CERTIFICATE).setSecureSettings(secureSettings).build();
+            settings = Settings.builder()
+                .put("xpack.ssl.certificate_authorities", certPath.toString())
+                .put("xpack.ssl.verification_mode", VerificationMode.CERTIFICATE)
+                .setSecureSettings(secureSettings)
+                .build();
         }
         CommandLineHttpClient client = new CommandLineHttpClient(settings, environment);
         HttpResponse httpResponse = client.execute("GET", new URL("https://localhost:" + webServer.getPort() + "/test"), "u1",
@@ -74,11 +77,15 @@ public class CommandLineHttpClientTests extends ESTestCase {
     }
 
     private MockWebServer createMockWebServer() {
-        Path resource = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.jks");
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+        Path keyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
-        Settings settings =
-                Settings.builder().put("xpack.ssl.keystore.path", resource.toString()).setSecureSettings(secureSettings).build();
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
+        Settings settings = Settings.builder()
+            .put("xpack.ssl.key", keyPath.toString())
+            .put("xpack.ssl.certificate", certPath.toString())
+            .setSecureSettings(secureSettings)
+            .build();
         TestsSSLService sslService = new TestsSSLService(settings, environment);
         return new MockWebServer(sslService.sslContext(), false);
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java
index 26cd513ec78..23010e400a5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java
@@ -34,8 +34,8 @@ public abstract class GroupsResolverTestCase extends ESTestCase {
 
     @Before
     public void setUpLdapConnection() throws Exception {
-        Path truststore = getDataPath(trustPath());
-        this.ldapConnection = LdapTestUtils.openConnection(ldapUrl(), bindDN(), bindPassword(), truststore);
+        Path trustPath = getDataPath(trustPath());
+        this.ldapConnection = LdapTestUtils.openConnection(ldapUrl(), bindDN(), bindPassword(), trustPath);
     }
 
     @After
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapTestUtils.java
index 8bdfd02d2fc..966f2e3f549 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapTestUtils.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapTestUtils.java
@@ -67,6 +67,6 @@ public class LdapTestUtils {
             sslConfiguration = sslService.getSSLConfiguration("xpack.security.authc.realms.foo.ssl");
         }
         return LdapUtils.privilegedConnect(() -> new LDAPConnection(sslService.sslSocketFactory(sslConfiguration), options,
-                ldapurl.getHost(), ldapurl.getPort(), bindDN, bindPassword));
+            ldapurl.getHost(), ldapurl.getPort(), bindDN, bindPassword));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java
index 9d8fd1544f5..19b0d4e71bb 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java
@@ -54,7 +54,7 @@ public class LdapUserSearchSessionFactoryTests extends LdapTestCase {
 
     @Before
     public void init() throws Exception {
-        Path keystore = getDataPath("support/ADtrust.jks");
+        Path certPath = getDataPath("support/smb_ca.crt");
         Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         /*
          * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext.
@@ -63,10 +63,9 @@ public class LdapUserSearchSessionFactoryTests extends LdapTestCase {
          */
 
         globalSettings = Settings.builder()
-                .put("path.home", createTempDir())
-                .put("xpack.ssl.truststore.path", keystore)
-                .setSecureSettings(newSecureSettings("xpack.ssl.truststore.secure_password", "changeit"))
-                .build();
+            .put("path.home", createTempDir())
+            .put("xpack.ssl.certificate_authorities", certPath)
+            .build();
         sslService = new SSLService(globalSettings, env);
         threadPool = new TestThreadPool("LdapUserSearchSessionFactoryTests");
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
index c458a9c42ea..931acc1e79d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
@@ -26,22 +26,24 @@ import org.elasticsearch.xpack.core.common.socket.SocketAccess;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
 import org.elasticsearch.xpack.core.ssl.SSLClientAuth;
 import org.elasticsearch.xpack.security.LocalStateSecurity;
 
-import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.KeyManager;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.TrustManager;
 
-import java.io.InputStream;
 import java.net.InetSocketAddress;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.security.KeyStore;
 import java.security.SecureRandom;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
+import java.util.stream.Collectors;
 
-import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForStore;
+import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForPEMFiles;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
@@ -60,16 +62,16 @@ public class PkiAuthenticationTests extends SecuritySingleNodeTestCase {
         SSLClientAuth sslClientAuth = randomBoolean() ? SSLClientAuth.REQUIRED : SSLClientAuth.OPTIONAL;
 
         Settings.Builder builder = Settings.builder()
-                .put(super.nodeSettings())
-                .put("xpack.security.http.ssl.enabled", true)
-                .put("xpack.security.http.ssl.client_authentication", sslClientAuth)
-                .put("xpack.security.authc.realms.file.type", FileRealmSettings.TYPE)
-                .put("xpack.security.authc.realms.file.order", "0")
-                .put("xpack.security.authc.realms.pki1.type", PkiRealmSettings.TYPE)
-                .put("xpack.security.authc.realms.pki1.order", "1")
-                .put("xpack.security.authc.realms.pki1.truststore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks"))
-                .put("xpack.security.authc.realms.pki1.files.role_mapping", getDataPath("role_mapping.yml"));
+            .put(super.nodeSettings())
+            .put("xpack.security.http.ssl.enabled", true)
+            .put("xpack.security.http.ssl.client_authentication", sslClientAuth)
+            .put("xpack.security.authc.realms.file.type", FileRealmSettings.TYPE)
+            .put("xpack.security.authc.realms.file.order", "0")
+            .put("xpack.security.authc.realms.pki1.type", PkiRealmSettings.TYPE)
+            .put("xpack.security.authc.realms.pki1.order", "1")
+            .put("xpack.security.authc.realms.pki1.certificate_authorities",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .put("xpack.security.authc.realms.pki1.files.role_mapping", getDataPath("role_mapping.yml"));
 
         SecuritySettingsSource.addSecureSettings(builder, secureSettings ->
                 secureSettings.setString("xpack.security.authc.realms.pki1.truststore.secure_password", "truststore-testnode-only"));
@@ -90,7 +92,13 @@ public class PkiAuthenticationTests extends SecuritySingleNodeTestCase {
 
     public void testTransportClientCanAuthenticateViaPki() {
         Settings.Builder builder = Settings.builder();
-        addSSLSettingsForStore(builder, "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks", "testnode");
+        addSSLSettingsForPEMFiles(
+            builder,
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem",
+            "testnode",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
+            Arrays.asList
+                ("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         try (TransportClient client = createTransportClient(builder.build())) {
             client.addTransportAddress(randomFrom(node().injector().getInstance(Transport.class).boundAddress().boundAddresses()));
             IndexResponse response = client.prepareIndex("foo", "bar").setSource("pki", "auth").get();
@@ -113,7 +121,11 @@ public class PkiAuthenticationTests extends SecuritySingleNodeTestCase {
     }
 
     public void testRestAuthenticationViaPki() throws Exception {
-        SSLContext context = getRestSSLContext("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks", "testnode");
+        SSLContext context = getRestSSLContext("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem",
+            "testnode",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt",
+                "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         try (CloseableHttpClient client = HttpClients.custom().setSSLContext(context).build()) {
             HttpPut put = new HttpPut(getNodeUrl() + "foo");
             try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(put))) {
@@ -124,7 +136,10 @@ public class PkiAuthenticationTests extends SecuritySingleNodeTestCase {
     }
 
     public void testRestAuthenticationFailure() throws Exception {
-        SSLContext context = getRestSSLContext("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks", "testclient");
+        SSLContext context = getRestSSLContext("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem",
+            "testclient", "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt",
+                "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         try (CloseableHttpClient client = HttpClients.custom().setSSLContext(context).build()) {
             HttpPut put = new HttpPut(getNodeUrl() + "foo");
             try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(put))) {
@@ -135,21 +150,13 @@ public class PkiAuthenticationTests extends SecuritySingleNodeTestCase {
         }
     }
 
-    private SSLContext getRestSSLContext(String keystoreResourcePath, String password) throws Exception {
+    private SSLContext getRestSSLContext(String keyPath, String password, String certPath, List<String> trustedCertPaths) throws Exception {
         SSLContext context = SSLContext.getInstance("TLS");
-        KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-        Path store = getDataPath(keystoreResourcePath);
-        KeyStore ks;
-        try (InputStream in = Files.newInputStream(store)) {
-            ks = KeyStore.getInstance("jks");
-            ks.load(in, password.toCharArray());
-        }
-
-        kmf.init(ks, password.toCharArray());
-        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-        tmf.init(ks);
-        context.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
-
+        TrustManager tm = CertParsingUtils.trustManager(CertParsingUtils.readCertificates(trustedCertPaths.stream().map(p -> getDataPath
+            (p)).collect(Collectors.toList())));
+        KeyManager km = CertParsingUtils.keyManager(CertParsingUtils.readCertificates(Collections.singletonList(getDataPath
+            (certPath))), PemUtils.readPrivateKey(getDataPath(keyPath), password::toCharArray), password.toCharArray());
+        context.init(new KeyManager[]{km}, new TrustManager[]{tm}, new SecureRandom());
         return context;
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
index 7f79ae35ada..5a7015a4e8d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
@@ -128,6 +128,8 @@ public class SamlAuthenticatorTests extends SamlTestCase {
 
     @BeforeClass
     public static void init() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, there is no DOM XMLSignature Factory so we can't sign XML documents", inFipsJvm());
+        // TODO: Refactor the signing to use org.opensaml.xmlsec.signature.support.Signer so that we can run the tests
         SamlUtils.initialize(Loggers.getLogger(SamlAuthenticatorTests.class));
         // Initialise Apache XML security so that the signDoc methods work correctly.
         Init.init();
@@ -218,7 +220,7 @@ public class SamlAuthenticatorTests extends SamlTestCase {
                 "<saml2:Issuer xmlns:saml2=\"urn:oasis:names:tc:SAML:2.0:assertion\">" +
                 IDP_ENTITY_ID + "</saml2:Issuer>" +
                 "<saml2p:Status><saml2p:StatusCode Value=\"urn:oasis:names:tc:SAML:2.0:status:Success\"/></saml2p:Status>" +
-                "</saml2p:Response>");
+            "</saml2p:Response>");
         final ElasticsearchSecurityException exception = expectSamlException(() -> authenticator.authenticate(token));
         assertThat(exception.getMessage(), containsString("No assertions found in SAML response"));
         assertThat(exception.getCause(), nullValue());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommandTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommandTests.java
index cf41673b86b..e239c8706b9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommandTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommandTests.java
@@ -296,6 +296,7 @@ public class SamlMetadataCommandTests extends SamlTestCase {
     }
 
     public void testSigningMetadataWithPfx() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, PKCS12 keystores are not usable", inFipsJvm());
         final Path certPath = getDataPath("saml.crt");
         final Path keyPath = getDataPath("saml.key");
         final Path p12Path = getDataPath("saml.p12");
@@ -355,6 +356,7 @@ public class SamlMetadataCommandTests extends SamlTestCase {
     }
 
     public void testSigningMetadataWithPasswordProtectedPfx() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, PKCS12 keystores are not usable", inFipsJvm());
         final Path certPath = getDataPath("saml.crt");
         final Path keyPath = getDataPath("saml.key");
         final Path p12Path = getDataPath("saml_with_password.p12");
@@ -393,11 +395,13 @@ public class SamlMetadataCommandTests extends SamlTestCase {
     public void testErrorSigningMetadataWithWrongPassword() throws Exception {
         final Path certPath = getDataPath("saml.crt");
         final Path keyPath = getDataPath("saml.key");
-        final Path p12Path = getDataPath("saml_with_password.p12");
+        final Path signingKeyPath = getDataPath("saml_with_password.key");
         final SamlMetadataCommand command = new SamlMetadataCommand((e) -> randomFrom(keyStore, null));
         final OptionSet options = command.getParser().parse(new String[]{
-                "-signing-bundle", p12Path.toString(),
-                "-signing-key-password", "wrong_password"
+            "-signing-cert", certPath.toString(),
+            "-signing-key", signingKeyPath.toString(),
+            "-signing-key-password", "wrongpassword"
+
         });
 
         final boolean useSigningCredentials = randomBoolean();
@@ -422,7 +426,7 @@ public class SamlMetadataCommandTests extends SamlTestCase {
         final UserException userException = expectThrows(UserException.class, () -> command.possiblySignDescriptor(terminal, options,
                 descriptor, env));
         assertThat(userException.getMessage(), containsString("Unable to create metadata document"));
-        assertThat(terminal.getOutput(), containsString("keystore password was incorrect"));
+        assertThat(terminal.getOutput(), containsString("Error parsing Private Key from"));
     }
 
     public void testSigningMetadataWithPem() throws Exception {
@@ -473,7 +477,7 @@ public class SamlMetadataCommandTests extends SamlTestCase {
         final OptionSet options = command.getParser().parse(new String[]{
                 "-signing-cert", certPath.toString(),
                 "-signing-key", signingKeyPath.toString(),
-                "-signing-key-password", "saml"
+            "-signing-key-password", "saml"
 
         });
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java
index 6dc9c021fc8..980abc46831 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java
@@ -105,13 +105,17 @@ public class SamlRealmTests extends SamlTestCase {
         final Path path = getDataPath("idp1.xml");
         final String body = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
         final MockSecureSettings mockSecureSettings = new MockSecureSettings();
-        mockSecureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        mockSecureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         final Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"))
-                .put("path.home", createTempDir())
-                .setSecureSettings(mockSecureSettings)
-                .build();
+            .put("xpack.ssl.key",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+            .put("xpack.ssl.certificate",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .put("xpack.ssl.certificate_authorities",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .put("path.home", createTempDir())
+            .setSecureSettings(mockSecureSettings)
+            .build();
         TestsSSLService sslService = new TestsSSLService(settings, TestEnvironment.newEnvironment(settings));
         try (MockWebServer proxyServer = new MockWebServer(sslService.sslContext(Settings.EMPTY), false)) {
             proxyServer.start();
@@ -563,17 +567,21 @@ public class SamlRealmTests extends SamlTestCase {
 
     private Settings.Builder buildSettings(String idpMetaDataPath) {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString(REALM_SETTINGS_PREFIX + ".ssl.keystore.secure_password", "testnode");
+        secureSettings.setString(REALM_SETTINGS_PREFIX + ".ssl.secure_key_passphrase", "testnode");
         return Settings.builder()
-                .put(REALM_SETTINGS_PREFIX + ".ssl.verification_mode", "certificate")
-                .put(REALM_SETTINGS_PREFIX + ".ssl.keystore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"))
-                .put(REALM_SETTINGS_PREFIX + ".type", "saml")
-                .put(REALM_SETTINGS_PREFIX + "." + SamlRealmSettings.IDP_METADATA_PATH.getKey(), idpMetaDataPath)
-                .put(REALM_SETTINGS_PREFIX + "." + SamlRealmSettings.IDP_ENTITY_ID.getKey(), TEST_IDP_ENTITY_ID)
-                .put(REALM_SETTINGS_PREFIX + "." + SamlRealmSettings.IDP_METADATA_HTTP_REFRESH.getKey(), METADATA_REFRESH + "ms")
-                .put("path.home", createTempDir())
-                .setSecureSettings(secureSettings);
+            .put(REALM_SETTINGS_PREFIX + ".ssl.verification_mode", "certificate")
+            .put(REALM_SETTINGS_PREFIX + ".ssl.key",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+            .put(REALM_SETTINGS_PREFIX + ".ssl.certificate",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .put(REALM_SETTINGS_PREFIX + ".ssl.certificate_authorities",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .put(REALM_SETTINGS_PREFIX + ".type", "saml")
+            .put(REALM_SETTINGS_PREFIX + "." + SamlRealmSettings.IDP_METADATA_PATH.getKey(), idpMetaDataPath)
+            .put(REALM_SETTINGS_PREFIX + "." + SamlRealmSettings.IDP_ENTITY_ID.getKey(), TEST_IDP_ENTITY_ID)
+            .put(REALM_SETTINGS_PREFIX + "." + SamlRealmSettings.IDP_METADATA_HTTP_REFRESH.getKey(), METADATA_REFRESH + "ms")
+            .put("path.home", createTempDir())
+            .setSecureSettings(secureSettings);
     }
 
     private RealmConfig realmConfigFromRealmSettings(Settings realmSettings) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java
index bac5e0b3f50..abd5768bebe 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java
@@ -43,11 +43,10 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.concurrent.CountDownLatch;
 
-import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForStore;
+import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForPEMFiles;
 import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.is;
 
 public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase {
     private static int randomClientPort;
@@ -66,25 +65,18 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
     protected Settings nodeSettings(int nodeOrdinal) {
         Settings.Builder settingsBuilder = Settings.builder();
         String randomClientPortRange = randomClientPort + "-" + (randomClientPort+100);
-
-        Path store;
-        try {
-            store = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
-            assertThat(Files.exists(store), is(true));
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
         settingsBuilder.put(super.nodeSettings(nodeOrdinal))
-                       .put("transport.profiles.client.xpack.security.ssl.truststore.path", store) // settings for client truststore
-                       .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
-                       .put("transport.profiles.client.xpack.security.type", "client")
-                       .put("transport.profiles.client.port", randomClientPortRange)
-                       // make sure this is "localhost", no matter if ipv4 or ipv6, but be consistent
-                       .put("transport.profiles.client.bind_host", "localhost")
-                       .put("xpack.security.audit.enabled", false)
-                       .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
-                       .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false);
+            .putList("transport.profiles.client.xpack.security.ssl.certificate_authorities",
+                Arrays.asList(certPath.toString())) // settings for client truststore
+            .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
+            .put("transport.profiles.client.xpack.security.type", "client")
+            .put("transport.profiles.client.port", randomClientPortRange)
+            // make sure this is "localhost", no matter if ipv4 or ipv6, but be consistent
+            .put("transport.profiles.client.bind_host", "localhost")
+            .put("xpack.security.audit.enabled", false)
+            .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
+            .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false);
         if (randomBoolean()) {
             settingsBuilder.put("transport.profiles.default.xpack.security.type", "node"); // this is default lets set it randomly
         }
@@ -120,7 +112,12 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
                 //.put("xpack.ml.autodetect_process", false);
         Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(
             LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class);
-        addSSLSettingsForStore(nodeSettings, "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks", "testnode");
+        addSSLSettingsForPEMFiles(
+            nodeSettings,
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem",
+            "testnode",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         try (Node node = new MockNode(nodeSettings.build(), mockPlugins)) {
             node.start();
             ensureStableCluster(cluster().size() + 1);
@@ -159,7 +156,12 @@ public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase
                 //.put("xpack.ml.autodetect_process", false);
         Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(
             LocalStateSecurity.class, TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class);
-        addSSLSettingsForStore(nodeSettings, "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks", "testnode");
+        addSSLSettingsForPEMFiles(
+            nodeSettings,
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem",
+            "testnode",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         try (Node node = new MockNode(nodeSettings.build(), mockPlugins)) {
             node.start();
             TransportService instance = node.injector().getInstance(TransportService.class);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IPHostnameVerificationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IPHostnameVerificationTests.java
index f03a4255b7f..bc674ae1aa0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IPHostnameVerificationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IPHostnameVerificationTests.java
@@ -21,7 +21,8 @@ import static org.hamcrest.CoreMatchers.is;
 
 // TODO delete this test?
 public class IPHostnameVerificationTests extends SecurityIntegTestCase {
-    Path keystore;
+    private Path certPath;
+    private Path keyPath;
 
     @Override
     protected boolean transportSSLEnabled() {
@@ -46,36 +47,37 @@ public class IPHostnameVerificationTests extends SecurityIntegTestCase {
                 .putList("discovery.zen.ping.unicast.hosts", newUnicastAddresses);
 
         try {
-            //This keystore uses a cert with a CN of "Elasticsearch Test Node" and IPv4+IPv6 ip addresses as SubjectAlternativeNames
-            keystore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.jks");
-            assertThat(Files.exists(keystore), is(true));
+            //Use a cert with a CN of "Elasticsearch Test Node" and IPv4+IPv6 ip addresses as SubjectAlternativeNames
+            certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.crt");
+            keyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.pem");
+            assertThat(Files.exists(certPath), is(true));
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
 
         SecuritySettingsSource.addSecureSettings(settingsBuilder, secureSettings -> {
-            secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode-ip-only");
-            secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode-ip-only");
+            secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode-ip-only");
         });
-        return settingsBuilder.put("xpack.ssl.keystore.path", keystore.toAbsolutePath()) // settings for client truststore
-                .put("xpack.ssl.truststore.path", keystore.toAbsolutePath()) // settings for client truststore
-                .put(TcpTransport.BIND_HOST.getKey(), "127.0.0.1")
-                .put("network.host", "127.0.0.1")
-                .put("xpack.ssl.client_authentication", SSLClientAuth.NONE)
-                .put("xpack.ssl.verification_mode", "full")
-                .build();
+        return settingsBuilder.put("xpack.ssl.key", keyPath.toAbsolutePath())
+            .put("xpack.ssl.certificate", certPath.toAbsolutePath())
+            .put("xpack.ssl.certificate_authorities", certPath.toAbsolutePath())
+            .put(TcpTransport.BIND_HOST.getKey(), "127.0.0.1")
+            .put("network.host", "127.0.0.1")
+            .put("xpack.ssl.client_authentication", SSLClientAuth.NONE)
+            .put("xpack.ssl.verification_mode", "full")
+            .build();
     }
 
     @Override
     protected Settings transportClientSettings() {
         Settings clientSettings = super.transportClientSettings();
         return Settings.builder().put(clientSettings.filter(k -> k.startsWith("xpack.ssl.") == false))
-                .put("xpack.ssl.verification_mode", "certificate")
-                .put("xpack.ssl.keystore.path", keystore.toAbsolutePath())
-                .put("xpack.ssl.keystore.password", "testnode-ip-only")
-                .put("xpack.ssl.truststore.path", keystore.toAbsolutePath())
-                .put("xpack.ssl.truststore.password", "testnode-ip-only")
-                .build();
+            .put("xpack.ssl.verification_mode", "certificate")
+            .put("xpack.ssl.key", keyPath.toAbsolutePath())
+            .put("xpack.ssl.certificate", certPath.toAbsolutePath())
+            .put("xpack.ssl.key_passphrase", "testnode-ip-only")
+            .put("xpack.ssl.certificate_authorities", certPath)
+            .build();
     }
 
     public void testTransportClientConnectionWorksWithIPOnlyHostnameVerification() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
index ad64dea79a5..23ca3c1fe9f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
@@ -39,17 +39,21 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
 
     private SSLService sslService;
     private Environment env;
-
+    private Path testnodeCert;
+    private Path testnodeKey;
     @Before
     public void createSSLService() throws Exception {
-        Path testNodeStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
+        testnodeCert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+        testnodeKey = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
+
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", testNodeStore)
-                .put("path.home", createTempDir())
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.ssl.key", testnodeKey)
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("path.home", createTempDir())
+            .setSecureSettings(secureSettings)
+            .build();
         env = TestEnvironment.newEnvironment(settings);
         sslService = new SSLService(settings, env);
     }
@@ -144,15 +148,11 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
     }
 
     public void testThatExceptionIsThrownWhenConfiguredWithoutSslKey() throws Exception {
-        MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.truststore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"))
-                .setSecureSettings(secureSettings)
-                .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
-                .put("path.home", createTempDir())
-                .build();
+            .put("xpack.ssl.certificate_authorities", testnodeCert)
+            .put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true)
+            .put("path.home", createTempDir())
+            .build();
         env = TestEnvironment.newEnvironment(settings);
         sslService = new SSLService(settings, env);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
@@ -163,13 +163,13 @@ public class SecurityNetty4HttpServerTransportTests extends ESTestCase {
 
     public void testNoExceptionWhenConfiguredWithoutSslKeySSLDisabled() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.truststore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"))
-                .setSecureSettings(secureSettings)
-                .put("path.home", createTempDir())
-                .build();
+            .put("xpack.ssl.key", testnodeKey)
+            .put("xpack.ssl.certificate", testnodeCert)
+            .setSecureSettings(secureSettings)
+            .put("path.home", createTempDir())
+            .build();
         env = TestEnvironment.newEnvironment(settings);
         sslService = new SSLService(settings, env);
         SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(settings,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportTests.java
index f87ab36d3d5..e9d91f5bd2d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportTests.java
@@ -41,15 +41,17 @@ public class SecurityNetty4ServerTransportTests extends ESTestCase {
 
     @Before
     public void createSSLService() throws Exception {
-        Path testnodeStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
+        Path testnodeCert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+        Path testnodeKey = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.security.transport.ssl.enabled", true)
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .setSecureSettings(secureSettings)
-                .put("path.home", createTempDir())
-                .build();
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.ssl.key", testnodeKey)
+            .put("xpack.ssl.certificate", testnodeCert)
+            .setSecureSettings(secureSettings)
+            .put("path.home", createTempDir())
+            .build();
         env = TestEnvironment.newEnvironment(settings);
         sslService = new SSLService(settings, env);
     }
@@ -179,17 +181,18 @@ public class SecurityNetty4ServerTransportTests extends ESTestCase {
 
     public void testTransportSSLOverridesGlobalSSL() throws Exception {
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.security.transport.ssl.keystore.secure_password", "testnode");
-        secureSettings.setString("xpack.ssl.truststore.secure_password", "truststore-testnode-only");
+        secureSettings.setString("xpack.security.transport.ssl.secure_key_passphrase", "testnode");
         Settings.Builder builder = Settings.builder()
-                .put("xpack.security.transport.ssl.enabled", true)
-                .put("xpack.security.transport.ssl.keystore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"))
-                .put("xpack.security.transport.ssl.client_authentication", "none")
-                .put("xpack.ssl.truststore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks"))
-                .setSecureSettings(secureSettings)
-                .put("path.home", createTempDir());
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.transport.ssl.key",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+            .put("xpack.security.transport.ssl.certificate",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .put("xpack.security.transport.ssl.client_authentication", "none")
+            .put("xpack.ssl.certificate_authorities",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .setSecureSettings(secureSettings)
+            .put("path.home", createTempDir());
         Settings settings = builder.build();
         env = TestEnvironment.newEnvironment(settings);
         sslService = new SSLService(settings, env);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SslHostnameVerificationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SslHostnameVerificationTests.java
index 148453b5f84..c61b5782f75 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SslHostnameVerificationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SslHostnameVerificationTests.java
@@ -19,6 +19,7 @@ import org.elasticsearch.xpack.security.LocalStateSecurity;
 import java.net.InetSocketAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.containsString;
@@ -35,25 +36,33 @@ public class SslHostnameVerificationTests extends SecurityIntegTestCase {
         Settings settings = super.nodeSettings(nodeOrdinal);
         Settings.Builder settingsBuilder = Settings.builder();
         settingsBuilder.put(settings.filter(k -> k.startsWith("xpack.ssl.") == false), false);
-        Path keystore;
+        Path keyPath;
+        Path certPath;
+        Path nodeCertPath;
         try {
             /*
              * This keystore uses a cert without any subject alternative names and a CN of "Elasticsearch Test Node No SAN"
              * that will not resolve to a DNS name and will always cause hostname verification failures
              */
-            keystore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.jks");
-            assert keystore != null;
-            assertThat(Files.exists(keystore), is(true));
+            keyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem");
+            certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt");
+            nodeCertPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+            assert keyPath != null;
+            assert certPath != null;
+            assert nodeCertPath != null;
+            assertThat(Files.exists(certPath), is(true));
+            assertThat(Files.exists(nodeCertPath), is(true));
+            assertThat(Files.exists(keyPath), is(true));
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
 
         SecuritySettingsSource.addSecureSettings(settingsBuilder, secureSettings -> {
-            secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode-no-subjaltname");
-            secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode-no-subjaltname");
+            secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode-no-subjaltname");
         });
-        return settingsBuilder.put("xpack.ssl.keystore.path", keystore.toAbsolutePath())
-                .put("xpack.ssl.truststore.path", keystore.toAbsolutePath())
+        return settingsBuilder.put("xpack.ssl.key", keyPath.toAbsolutePath())
+            .put("xpack.ssl.certificate", certPath.toAbsolutePath())
+            .putList("xpack.ssl.certificate_authorities", Arrays.asList(certPath.toString(), nodeCertPath.toString()))
                 // disable hostname verification as this test uses certs without a valid SAN or DNS in the CN
                 .put("xpack.ssl.verification_mode", "certificate")
                 .build();
@@ -61,22 +70,32 @@ public class SslHostnameVerificationTests extends SecurityIntegTestCase {
 
     @Override
     protected Settings transportClientSettings() {
-        Path keystore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.jks");
-        assert keystore != null;
+        Path keyPath;
+        Path certPath;
+        Path nodeCertPath;
+        try {
+            keyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem");
+            certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt");
+            nodeCertPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+            assert keyPath != null;
+            assert certPath != null;
+            assert nodeCertPath != null;
+            assertThat(Files.exists(certPath), is(true));
+            assertThat(Files.exists(nodeCertPath), is(true));
+            assertThat(Files.exists(keyPath), is(true));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
         Settings settings = super.transportClientSettings();
         // remove all ssl settings
         Settings.Builder builder = Settings.builder();
         builder.put(settings.filter( k -> k.startsWith("xpack.ssl.") == false), false);
 
         builder.put("xpack.ssl.verification_mode", "certificate")
-                .put("xpack.ssl.keystore.path", keystore.toAbsolutePath()) // settings for client keystore
-                .put("xpack.ssl.keystore.password", "testnode-no-subjaltname");
-
-        if (randomBoolean()) {
-            // randomly set the truststore, if not set the keystore should be used
-            builder.put("xpack.ssl.truststore.path", keystore.toAbsolutePath())
-                    .put("xpack.ssl.truststore.password", "testnode-no-subjaltname");
-        }
+            .put("xpack.ssl.key", keyPath.toAbsolutePath())
+            .put("xpack.ssl.key_passphrase", "testnode-no-subjaltname")
+            .put("xpack.ssl.certificate", certPath.toAbsolutePath())
+            .putList("xpack.ssl.certificate_authorities", Arrays.asList(certPath.toString(), nodeCertPath.toString()));
         return builder.build();
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLDriverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLDriverTests.java
index b98e4e0ce57..e1e05032014 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLDriverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLDriverTests.java
@@ -7,21 +7,21 @@ package org.elasticsearch.xpack.security.transport.nio;
 
 import org.elasticsearch.nio.InboundChannelBuffer;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
 
-import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.KeyManager;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLException;
-import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.TrustManager;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.security.KeyStore;
 import java.security.SecureRandom;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.function.Supplier;
 
 public class SSLDriverTests extends ESTestCase {
@@ -205,19 +205,16 @@ public class SSLDriverTests extends ESTestCase {
     }
 
     private SSLContext getSSLContext() throws Exception {
-        String relativePath = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks";
+        String certPath = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt";
+        String keyPath = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem";
         SSLContext sslContext;
-        try (InputStream in = Files.newInputStream(getDataPath(relativePath))) {
-            KeyStore keyStore = KeyStore.getInstance("jks");
-            keyStore.load(in, "testclient".toCharArray());
-            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-            tmf.init(keyStore);
-            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-            kmf.init(keyStore, "testclient".toCharArray());
-            sslContext = SSLContext.getInstance("TLSv1.2");
-            sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
-            return sslContext;
-        }
+        TrustManager tm = CertParsingUtils.trustManager(CertParsingUtils.readCertificates(Collections.singletonList(getDataPath
+            (certPath))));
+        KeyManager km = CertParsingUtils.keyManager(CertParsingUtils.readCertificates(Collections.singletonList(getDataPath
+            (certPath))), PemUtils.readPrivateKey(getDataPath(keyPath), "testclient"::toCharArray), "testclient".toCharArray());
+        sslContext = SSLContext.getInstance("TLSv1.2");
+        sslContext.init(new KeyManager[] { km }, new TrustManager[] { tm }, new SecureRandom());
+        return sslContext;
     }
 
     private void normalClose(SSLDriver sendDriver, SSLDriver receiveDriver) throws IOException {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SimpleSecurityNioTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SimpleSecurityNioTransportTests.java
index 9f33da7ae88..feca093e581 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SimpleSecurityNioTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SimpleSecurityNioTransportTests.java
@@ -55,15 +55,17 @@ import static org.hamcrest.Matchers.instanceOf;
 public class SimpleSecurityNioTransportTests extends AbstractSimpleTransportTestCase {
 
     private SSLService createSSLService() {
-        Path testnodeStore = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
+        Path testnodeCert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+        Path testnodeKey = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.security.transport.ssl.enabled", true)
-                .put("xpack.ssl.keystore.path", testnodeStore)
-                .setSecureSettings(secureSettings)
-                .put("path.home", createTempDir())
-                .build();
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.ssl.key", testnodeKey)
+            .put("xpack.ssl.certificate", testnodeCert)
+            .put("path.home", createTempDir())
+            .setSecureSettings(secureSettings)
+            .build();
         try {
             return new SSLService(settings, TestEnvironment.newEnvironment(settings));
         } catch (Exception e) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/EllipticCurveSSLTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/EllipticCurveSSLTests.java
index 42c5cd7c7ab..df49103a259 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/EllipticCurveSSLTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/EllipticCurveSSLTests.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.security.transport.ssl;
 
-import com.unboundid.util.ssl.TrustAllTrustManager;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -77,7 +76,8 @@ public class EllipticCurveSSLTests extends SecurityIntegTestCase {
         X509ExtendedKeyManager x509ExtendedKeyManager = CertParsingUtils.keyManager(certs, privateKey, new char[0]);
         SSLContext sslContext = SSLContext.getInstance("TLS");
         sslContext.init(new X509ExtendedKeyManager[] { x509ExtendedKeyManager },
-                new TrustManager[] { new TrustAllTrustManager(false) }, new SecureRandom());
+            new TrustManager[]{CertParsingUtils.trustManager(CertParsingUtils.readCertificates(Collections.singletonList(certPath)))},
+            new SecureRandom());
         SSLSocketFactory socketFactory = sslContext.getSocketFactory();
         NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().setTransport(true).get();
         TransportAddress address = randomFrom(response.getNodes()).getTransport().getAddress().publishAddress();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
index fa8fd00aeba..541e6606912 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
@@ -42,12 +42,13 @@ import java.nio.charset.StandardCharsets;
 import java.security.KeyStore;
 import java.security.SecureRandom;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 
-import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForStore;
+import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForPEMFiles;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
@@ -97,6 +98,7 @@ public class SslIntegrationTests extends SecurityIntegTestCase {
 
     // no SSL exception as this is the exception is returned when connecting
     public void testThatTransportClientUsingSSLv3ProtocolIsRejected() {
+        assumeFalse("Can't run in a FIPS JVM as SSLv3 SSLContext not available", inFipsJvm());
         try (TransportClient transportClient = new TestXPackTransportClient(Settings.builder()
                 .put(transportClientSettings())
                 .put("node.name", "programmatic_transport_client")
@@ -116,7 +118,11 @@ public class SslIntegrationTests extends SecurityIntegTestCase {
 
     public void testThatConnectionToHTTPWorks() throws Exception {
         Settings.Builder builder = Settings.builder();
-        addSSLSettingsForStore(builder, "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks", "testclient");
+        addSSLSettingsForPEMFiles(
+            builder, "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem",
+            "testclient",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         SSLService service = new SSLService(builder.build(), null);
 
         CredentialsProvider provider = new BasicCredentialsProvider();
@@ -135,6 +141,7 @@ public class SslIntegrationTests extends SecurityIntegTestCase {
     }
 
     public void testThatHttpUsingSSLv3IsRejected() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM as we can't even get an instance of SSL SSL Context", inFipsJvm());
         SSLContext sslContext = SSLContext.getInstance("SSL");
         TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
         factory.init((KeyStore) null);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslMultiPortTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslMultiPortTests.java
index 1d7ec67762b..d3ab5d092ab 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslMultiPortTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslMultiPortTests.java
@@ -20,10 +20,11 @@ import org.junit.BeforeClass;
 import java.net.InetAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.Collections;
 
 import static org.elasticsearch.test.SecuritySettingsSource.TEST_USER_NAME;
-import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForStore;
+import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForPEMFiles;
 import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.containsString;
@@ -42,9 +43,9 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
     /**
      * On each node sets up the following profiles:
      * <ul>
-     *     <li>default: testnode keystore. Requires client auth</li>
-     *     <li>client: testnode-client-profile keystore that only trusts the testclient cert. Requires client auth</li>
-     *     <li>no_client_auth: testnode keystore. Does not require client auth</li>
+     *     <li>default: testnode keypair. Requires client auth</li>
+     *     <li>client: testnode-client-profile profile  that only trusts the testclient cert. Requires client auth</li>
+     *     <li>no_client_auth: testnode keypair. Does not require client auth</li>
      * </ul>
      */
     @Override
@@ -52,26 +53,25 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
         String randomClientPortRange = randomClientPort + "-" + (randomClientPort+100);
         String randomNoClientAuthPortRange = randomNoClientAuthPort + "-" + (randomNoClientAuthPort+100);
 
-        Path store;
+        Path trustCert;
         try {
-            store = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.jks");
-            assertThat(Files.exists(store), is(true));
+            trustCert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.crt");
+            assertThat(Files.exists(trustCert), is(true));
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
 
         Settings settings = Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                // client set up here
-                .put("transport.profiles.client.port", randomClientPortRange)
-                // make sure this is "localhost", no matter if ipv4 or ipv6, but be consistent
-                .put("transport.profiles.client.bind_host", "localhost")
-                .put("transport.profiles.client.xpack.security.ssl.truststore.path", store.toAbsolutePath())
-                .put("transport.profiles.client.xpack.security.ssl.truststore.password", "testnode-client-profile")
-                .put("transport.profiles.no_client_auth.port", randomNoClientAuthPortRange)
-                .put("transport.profiles.no_client_auth.bind_host", "localhost")
-                .put("transport.profiles.no_client_auth.xpack.security.ssl.client_authentication", SSLClientAuth.NONE)
-                .build();
+            .put(super.nodeSettings(nodeOrdinal))
+            // client set up here
+            .put("transport.profiles.client.port", randomClientPortRange)
+            // make sure this is "localhost", no matter if ipv4 or ipv6, but be consistent
+            .put("transport.profiles.client.bind_host", "localhost")
+            .put("transport.profiles.client.xpack.security.ssl.certificate_authorities", trustCert.toAbsolutePath())
+            .put("transport.profiles.no_client_auth.port", randomNoClientAuthPortRange)
+            .put("transport.profiles.no_client_auth.bind_host", "localhost")
+            .put("transport.profiles.no_client_auth.xpack.security.ssl.client_authentication", SSLClientAuth.NONE)
+            .build();
         logger.info("node {} settings:\n{}", nodeOrdinal, settings);
         return settings;
     }
@@ -140,15 +140,18 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
     }
 
     /**
-     * Uses a transport client with a custom keystore; this keystore testclient-client-profile.jks trusts the testnode
+     * Uses a transport client with a custom key pair; TransportClient only trusts the testnode
      * certificate and had its own self signed certificate. This test connects to the client profile, which is only
      * set to trust the testclient-client-profile certificate so the connection should always succeed
      */
     public void testThatProfileTransportClientCanConnectToClientProfile() throws Exception {
         Settings.Builder builder = Settings.builder();
-        addSSLSettingsForStore(builder,
-                "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.jks",
-                "testclient-client-profile");
+        addSSLSettingsForPEMFiles(
+            builder,
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.pem",
+            "testclient-client-profile",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         try (TransportClient transportClient = createTransportClient(builder.build())) {
             transportClient.addTransportAddress(new TransportAddress(InetAddress.getLoopbackAddress(), getProfilePort("client")));
             assertGreenClusterState(transportClient);
@@ -156,16 +159,19 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
     }
 
     /**
-     * Uses a transport client with a custom keystore; this keystore testclient-client-profile.jks trusts the testnode
+     * Uses a transport client with a custom key pair; TransportClient only trusts the testnode
      * certificate and had its own self signed certificate. This test connects to the no_client_auth profile, which
      * uses a truststore that does not trust the testclient-client-profile certificate but does not require client
      * authentication
      */
     public void testThatProfileTransportClientCanConnectToNoClientAuthProfile() throws Exception {
         Settings.Builder builder = Settings.builder();
-        addSSLSettingsForStore(builder,
-                "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.jks",
-                "testclient-client-profile");
+        addSSLSettingsForPEMFiles(
+            builder,
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.pem",
+            "testclient-client-profile",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         try (TransportClient transportClient = createTransportClient(builder.build())) {
             transportClient.addTransportAddress(new TransportAddress(InetAddress.getLoopbackAddress(),
                     getProfilePort("no_client_auth")));
@@ -174,16 +180,19 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
     }
 
     /**
-     * Uses a transport client with a custom keystore; this keystore testclient-client-profile.jks trusts the testnode
+     * Uses a transport client with a custom key pair; TransportClient only trusts the testnode
      * certificate and had its own self signed certificate. This test connects to the default profile, which
      * uses a truststore that does not trust the testclient-client-profile certificate and requires client authentication
      * so the connection should always fail
      */
     public void testThatProfileTransportClientCannotConnectToDefaultProfile() throws Exception {
         Settings.Builder builder = Settings.builder();
-        addSSLSettingsForStore(builder,
-                "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.jks",
-                "testclient-client-profile");
+        addSSLSettingsForPEMFiles(
+            builder,
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.pem",
+            "testclient-client-profile",
+            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.crt",
+            Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
         try (TransportClient transportClient = createTransportClient(builder.build())) {
             TransportAddress transportAddress = randomFrom(internalCluster().getInstance(Transport.class).boundAddress().boundAddresses());
             transportClient.addTransportAddress(transportAddress);
@@ -253,19 +262,17 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
     }
 
     /**
-     * Uses a transport client with a custom truststore; this truststore truststore-testnode-only only trusts the testnode
-     * certificate and contains no other certification. This test connects to the no_client_auth profile, which uses
-     * the testnode certificate and does not require to present a certificate, so this connection should always succeed
+     * Uses a transport client that only trusts the testnode certificate. This test connects to the no_client_auth profile,
+     * which uses  the testnode certificate and does not require to present a certificate, so this connection should always succeed
      */
     public void testThatTransportClientWithOnlyTruststoreCanConnectToNoClientAuthProfile() throws Exception {
         Settings settings = Settings.builder()
-                .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
-                .put("cluster.name", internalCluster().getClusterName())
-                .put("xpack.security.transport.ssl.enabled", true)
-                .put("xpack.ssl.truststore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks"))
-                .put("xpack.ssl.truststore.password", "truststore-testnode-only")
-                .build();
+            .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
+            .put("cluster.name", internalCluster().getClusterName())
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.ssl.certificate_authorities",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .build();
         try (TransportClient transportClient = new TestXPackTransportClient(settings,
                                                                             Collections.singletonList(LocalStateSecurity.class))) {
             transportClient.addTransportAddress(new TransportAddress(InetAddress.getLoopbackAddress(),
@@ -274,21 +281,19 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
     }
 
     /**
-     * Uses a transport client with a custom truststore; this truststore truststore-testnode-only only trusts the testnode
-     * certificate and contains no other certification. This test connects to the client profile, which uses
+     * Uses a transport client that only trusts the testnode certificate. This test connects to the client profile, which uses
      * the testnode certificate and requires the client to present a certificate, so this connection will never work as
      * the client has no certificate to present
      */
     public void testThatTransportClientWithOnlyTruststoreCannotConnectToClientProfile() throws Exception {
         Settings settings = Settings.builder()
-                .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
-                .put("cluster.name", internalCluster().getClusterName())
-                .put("xpack.security.transport.ssl.enabled", true)
-                .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
-                .put("xpack.ssl.truststore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks"))
-                .put("xpack.ssl.truststore.password", "truststore-testnode-only")
-                .build();
+            .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
+            .put("cluster.name", internalCluster().getClusterName())
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
+            .put("xpack.ssl.certificate_authorities",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .build();
         try (TransportClient transportClient = new TestXPackTransportClient(settings,
                                                                             Collections.singletonList(LocalStateSecurity.class))) {
             transportClient.addTransportAddress(new TransportAddress(InetAddress.getLoopbackAddress(), getProfilePort("client")));
@@ -300,21 +305,19 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
     }
 
     /**
-     * Uses a transport client with a custom truststore; this truststore truststore-testnode-only only trusts the testnode
-     * certificate and contains no other certification. This test connects to the default profile, which uses
+     * Uses a transport client that only trusts the testnode certificate. This test connects to the default profile, which uses
      * the testnode certificate and requires the client to present a certificate, so this connection will never work as
      * the client has no certificate to present
      */
     public void testThatTransportClientWithOnlyTruststoreCannotConnectToDefaultProfile() throws Exception {
         Settings settings = Settings.builder()
-                .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
-                .put("cluster.name", internalCluster().getClusterName())
-                .put("xpack.security.transport.ssl.enabled", true)
-                .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
-                .put("xpack.ssl.truststore.path",
-                        getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks"))
-                .put("xpack.ssl.truststore.password", "truststore-testnode-only")
-                .build();
+            .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
+            .put("cluster.name", internalCluster().getClusterName())
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
+            .put("xpack.ssl.certificate_authorities",
+                getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .build();
         try (TransportClient transportClient = new TestXPackTransportClient(settings,
                                                                             Collections.singletonList(LocalStateSecurity.class))) {
             transportClient.addTransportAddress(randomFrom(internalCluster().getInstance(Transport.class).boundAddress().boundAddresses()));
@@ -332,11 +335,11 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
      */
     public void testThatSSLTransportClientWithNoTruststoreCannotConnectToDefaultProfile() throws Exception {
         Settings settings = Settings.builder()
-                .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
-                .put("cluster.name", internalCluster().getClusterName())
-                .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
-                .put("xpack.security.transport.ssl.enabled", true)
-                .build();
+            .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
+            .put("cluster.name", internalCluster().getClusterName())
+            .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .build();
         try (TransportClient transportClient = new TestXPackTransportClient(settings,
                                                                             Collections.singletonList(LocalStateSecurity.class))) {
             transportClient.addTransportAddress(randomFrom(internalCluster().getInstance(Transport.class).boundAddress().boundAddresses()));
@@ -354,11 +357,11 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
      */
     public void testThatSSLTransportClientWithNoTruststoreCannotConnectToClientProfile() throws Exception {
         Settings settings = Settings.builder()
-                .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
-                .put("cluster.name", internalCluster().getClusterName())
-                .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
-                .put("xpack.security.transport.ssl.enabled", true)
-                .build();
+            .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
+            .put("cluster.name", internalCluster().getClusterName())
+            .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .build();
         try (TransportClient transportClient = new TestXPackTransportClient(settings,
                                                                             Collections.singletonList(LocalStateSecurity.class))) {
             transportClient.addTransportAddress(new TransportAddress(InetAddress.getLoopbackAddress(), getProfilePort("client")));
@@ -376,11 +379,11 @@ public class SslMultiPortTests extends SecurityIntegTestCase {
      */
     public void testThatSSLTransportClientWithNoTruststoreCannotConnectToNoClientAuthProfile() throws Exception {
         Settings settings = Settings.builder()
-                .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
-                .put("cluster.name", internalCluster().getClusterName())
-                .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
-                .put("xpack.security.transport.ssl.enabled", true)
-                .build();
+            .put(SecurityField.USER_SETTING.getKey(), TEST_USER_NAME + ":" + TEST_PASSWORD)
+            .put("cluster.name", internalCluster().getClusterName())
+            .put("xpack.ssl.client_authentication", SSLClientAuth.REQUIRED)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .build();
         try (TransportClient transportClient = new TestXPackTransportClient(settings,
                                                                             Collections.singletonList(LocalStateSecurity.class))) {
             transportClient.addTransportAddress(new TransportAddress(InetAddress.getLoopbackAddress(),
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslNullCipherTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslNullCipherTests.java
index fb5d567bb36..7427c5a67e9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslNullCipherTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslNullCipherTests.java
@@ -10,12 +10,18 @@ import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.SecurityIntegTestCase;
+import org.junit.BeforeClass;
 
 /**
  * An extremely simple test that shows SSL will work with a cipher that does not perform encryption
  */
 public class SslNullCipherTests extends SecurityIntegTestCase {
 
+    @BeforeClass
+    public static void muteInFips() {
+        assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
+    }
+
     @Override
     public boolean transportSSLEnabled() {
         return true;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
index fdb66916884..d205c7cd933 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
@@ -22,20 +22,22 @@ import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.core.TestXPackTransportClient;
 import org.elasticsearch.xpack.core.security.SecurityField;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
 import org.elasticsearch.xpack.core.ssl.SSLClientAuth;
 import org.elasticsearch.xpack.security.LocalStateSecurity;
 
-import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.KeyManager;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.TrustManager;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.security.KeyStore;
 import java.security.SecureRandom;
 import java.security.cert.CertPathBuilderException;
+import java.util.Arrays;
+import java.util.Collections;
 
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.containsString;
@@ -74,7 +76,11 @@ public class SSLClientAuthTests extends SecurityIntegTestCase {
         } catch (IOException e) {
             Throwable t = ExceptionsHelper.unwrap(e, CertPathBuilderException.class);
             assertThat(t, instanceOf(CertPathBuilderException.class));
-            assertThat(t.getMessage(), containsString("unable to find valid certification path to requested target"));
+            if (inFipsJvm()) {
+                assertThat(t.getMessage(), containsString("Unable to find certificate chain"));
+            } else {
+                assertThat(t.getMessage(), containsString("unable to find valid certification path to requested target"));
+            }
         }
     }
 
@@ -89,24 +95,27 @@ public class SSLClientAuthTests extends SecurityIntegTestCase {
     }
 
     public void testThatTransportWorksWithoutSslClientAuth() throws IOException {
-        // specify an arbitrary keystore, that does not include the certs needed to connect to the transport protocol
-        Path store = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.jks");
+        // specify an arbitrary key and certificate - not the certs needed to connect to the transport protocol
+        Path keyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.pem");
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.crt");
+        Path nodeCertPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
 
-        if (Files.notExists(store)) {
-            throw new ElasticsearchException("store path doesn't exist");
+        if (Files.notExists(keyPath) || Files.notExists(certPath)) {
+            throw new ElasticsearchException("key or certificate path doesn't exist");
         }
 
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testclient-client-profile");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testclient-client-profile");
         Settings settings = Settings.builder()
-                .put("xpack.security.transport.ssl.enabled", true)
-                .put("xpack.ssl.client_authentication", SSLClientAuth.NONE)
-                .put("xpack.ssl.keystore.path", store)
-                .setSecureSettings(secureSettings)
-                .put("cluster.name", internalCluster().getClusterName())
-                .put(SecurityField.USER_SETTING.getKey(),
-                        transportClientUsername() + ":" + new String(transportClientPassword().getChars()))
-                .build();
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.ssl.client_authentication", SSLClientAuth.NONE)
+            .put("xpack.ssl.key", keyPath)
+            .put("xpack.ssl.certificate", certPath)
+            .put("xpack.ssl.certificate_authorities", nodeCertPath)
+            .setSecureSettings(secureSettings)
+            .put("cluster.name", internalCluster().getClusterName())
+            .put(SecurityField.USER_SETTING.getKey(), transportClientUsername() + ":" + new String(transportClientPassword().getChars()))
+            .build();
         try (TransportClient client = new TestXPackTransportClient(settings, LocalStateSecurity.class)) {
             Transport transport = internalCluster().getDataNodeInstance(Transport.class);
             TransportAddress transportAddress = transport.boundAddress().publishAddress();
@@ -117,19 +126,19 @@ public class SSLClientAuthTests extends SecurityIntegTestCase {
     }
 
     private SSLContext getSSLContext() {
-        try (InputStream in =
-                     Files.newInputStream(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks"))) {
-            KeyStore keyStore = KeyStore.getInstance("jks");
-            keyStore.load(in, "testclient".toCharArray());
-            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-            tmf.init(keyStore);
-            KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-            kmf.init(keyStore, "testclient".toCharArray());
+        try {
+            String certPath = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt";
+            String nodeCertPath = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt";
+            String keyPath = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem";
+            TrustManager tm = CertParsingUtils.trustManager(CertParsingUtils.readCertificates(Arrays.asList(getDataPath
+                (certPath), getDataPath(nodeCertPath))));
+            KeyManager km = CertParsingUtils.keyManager(CertParsingUtils.readCertificates(Collections.singletonList(getDataPath
+                (certPath))), PemUtils.readPrivateKey(getDataPath(keyPath), "testclient"::toCharArray), "testclient".toCharArray());
             SSLContext context = SSLContext.getInstance("TLSv1.2");
-            context.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
+            context.init(new KeyManager[] { km }, new TrustManager[] { tm }, new SecureRandom());
             return context;
         } catch (Exception e) {
-            throw new ElasticsearchException("failed to initialize a TrustManagerFactory", e);
+            throw new ElasticsearchException("failed to initialize SSLContext", e);
         }
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
index 4269d8a78eb..03f963cc59c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
@@ -12,9 +12,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.transport.Transport;
-import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
 import org.elasticsearch.xpack.core.ssl.SSLConfiguration;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 
@@ -23,16 +21,12 @@ import javax.net.ssl.SSLSocket;
 import javax.net.ssl.SSLSocketFactory;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.SocketException;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
-import java.security.KeyStore;
-import java.security.cert.X509Certificate;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.concurrent.CountDownLatch;
 
 import static org.hamcrest.Matchers.containsString;
@@ -43,34 +37,51 @@ import static org.hamcrest.Matchers.is;
  */
 public class SSLReloadIntegTests extends SecurityIntegTestCase {
 
-    private Path nodeStorePath;
+    private Path nodeKeyPath;
+    private Path nodeCertPath;
+    private Path clientCertPath;
+    private Path updateableCertPath;
 
     @Override
     public Settings nodeSettings(int nodeOrdinal) {
-        //Node starts with testnode.jks
-        if (nodeStorePath == null) {
-            Path origPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
-            Path tempDir = createTempDir();
-            nodeStorePath = tempDir.resolve("testnode.jks");
-            try {
-                Files.copy(origPath, nodeStorePath);
-            } catch (IOException e) {
-                throw new ElasticsearchException("failed to copy keystore");
+        // Nodes start trusting testnode.crt and testclient.crt
+        Path origKeyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
+        Path origCertPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+        Path origClientCertPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt");
+        Path tempDir = createTempDir();
+        try {
+            if (nodeKeyPath == null) {
+                nodeKeyPath = tempDir.resolve("testnode.pem");
+                Files.copy(origKeyPath, nodeKeyPath);
             }
+            if (nodeCertPath == null) {
+                nodeCertPath = tempDir.resolve("testnode.crt");
+                Files.copy(origCertPath, nodeCertPath);
+            }
+            if (clientCertPath == null) {
+                clientCertPath = tempDir.resolve("testclient.crt");
+                Files.copy(origClientCertPath, clientCertPath);
+            }
+            // Placeholder trusted certificate that will be updated later on
+            if (updateableCertPath == null) {
+                updateableCertPath = tempDir.resolve("updateable.crt");
+                Files.copy(origCertPath, updateableCertPath);
+            }
+        } catch (IOException e) {
+            throw new ElasticsearchException("failed to copy key or certificate", e);
         }
+
         Settings settings = super.nodeSettings(nodeOrdinal);
         Settings.Builder builder = Settings.builder()
                 .put(settings.filter((s) -> s.startsWith("xpack.ssl.") == false));
 
-
-        SecuritySettingsSource.addSSLSettingsForStore(builder,
-            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks", "testnode");
-        builder.put("resource.reload.interval.high", "1s")
-                .put("xpack.ssl.keystore.path", nodeStorePath);
-
-        if (builder.get("xpack.ssl.truststore.path") != null) {
-            builder.put("xpack.ssl.truststore.path", nodeStorePath);
-        }
+        builder.put("path.home", createTempDir())
+            .put("xpack.ssl.key", nodeKeyPath)
+            .put("xpack.ssl.key_passphrase", "testnode")
+            .put("xpack.ssl.certificate", nodeCertPath)
+            .putList("xpack.ssl.certificate_authorities", Arrays.asList(nodeCertPath.toString(), clientCertPath.toString(),
+                updateableCertPath.toString()))
+            .put("resource.reload.interval.high", "1s");
 
         return builder.build();
     }
@@ -81,25 +92,27 @@ public class SSLReloadIntegTests extends SecurityIntegTestCase {
     }
 
     public void testThatSSLConfigurationReloadsOnModification() throws Exception {
-        Path keystorePath = createTempDir().resolve("testnode_updated.jks");
-        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.jks"), keystorePath);
-        X509Certificate certificate = CertParsingUtils.readX509Certificates(Collections.singletonList(getDataPath
-                ("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt")))[0];
+        Path keyPath = createTempDir().resolve("testnode_updated.pem");
+        Path certPath = createTempDir().resolve("testnode_updated.crt");
+        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem"), keyPath);
+        Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt"), certPath);
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
-        secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("path.home", createTempDir())
-                .put("xpack.ssl.keystore.path", keystorePath)
-                .put("xpack.ssl.truststore.path", nodeStorePath)
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("path.home", createTempDir())
+            .put("xpack.ssl.key", keyPath)
+            .put("xpack.ssl.certificate", certPath)
+            .putList("xpack.ssl.certificate_authorities", Arrays.asList(nodeCertPath.toString(), clientCertPath.toString(),
+                updateableCertPath.toString()))
+            .setSecureSettings(secureSettings)
+            .build();
         String node = randomFrom(internalCluster().getNodeNames());
         SSLService sslService = new SSLService(settings, TestEnvironment.newEnvironment(settings));
         SSLConfiguration sslConfiguration = sslService.getSSLConfiguration("xpack.ssl");
         SSLSocketFactory sslSocketFactory = sslService.sslSocketFactory(sslConfiguration);
         TransportAddress address = internalCluster()
-                .getInstance(Transport.class, node).boundAddress().publishAddress();
+            .getInstance(Transport.class, node).boundAddress().publishAddress();
+        // Fails as our nodes do not trust testnode_updated.crt
         try (SSLSocket socket = (SSLSocket) sslSocketFactory.createSocket(address.getAddress(), address.getPort())) {
             assertThat(socket.isConnected(), is(true));
             socket.startHandshake();
@@ -107,19 +120,11 @@ public class SSLReloadIntegTests extends SecurityIntegTestCase {
         } catch (SSLHandshakeException | SocketException expected) {
             logger.trace("expected exception", expected);
         }
-        KeyStore nodeStore = KeyStore.getInstance("jks");
-        try (InputStream in = Files.newInputStream(nodeStorePath)) {
-            nodeStore.load(in, "testnode".toCharArray());
-        }
-        nodeStore.setCertificateEntry("newcert", certificate);
-        Path path = nodeStorePath.getParent().resolve("updated.jks");
-        try (OutputStream out = Files.newOutputStream(path)) {
-            nodeStore.store(out, "testnode".toCharArray());
-        }
+        // Copy testnode_updated.crt to the placeholder updateable.crt so that the nodes will start trusting it now
         try {
-            Files.move(path, nodeStorePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+            Files.move(certPath, updateableCertPath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
         } catch (AtomicMoveNotSupportedException e) {
-            Files.move(path, nodeStorePath, StandardCopyOption.REPLACE_EXISTING);
+            Files.move(certPath, updateableCertPath, StandardCopyOption.REPLACE_EXISTING);
         }
         CountDownLatch latch = new CountDownLatch(1);
         assertBusy(() -> {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
index beebf928fcf..cf77ca975a4 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
@@ -72,6 +72,7 @@ public class SSLTrustRestrictionsTests extends SecurityIntegTestCase {
 
     @BeforeClass
     public static void setupCertificates() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, custom TrustManager implementations cannot be used.", inFipsJvm());
         configPath = createTempDir();
         Path caCertPath = PathUtils.get(SSLTrustRestrictionsTests.class.getResource
                 ("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca.crt").toURI());
diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ad.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ad.crt
new file mode 100644
index 00000000000..2dbb06c49e6
--- /dev/null
+++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ad.crt
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID1zCCAr+gAwIBAgIQWA24rVK7FopAgOHfEio/VjANBgkqhkiG9w0BAQsFADB+
+MRMwEQYKCZImiZPyLGQBGRYDY29tMR0wGwYKCZImiZPyLGQBGRYNZWxhc3RpY3Nl
+YXJjaDEUMBIGCgmSJomT8ixkARkWBHRlc3QxEjAQBgoJkiaJk/IsZAEZFgJhZDEe
+MBwGA1UEAxMVYWQtRUxBU1RJQ1NFQVJDSEFELUNBMB4XDTE0MDgyNzE2MjI0MloX
+DTI5MDgyNzE2MzI0MlowfjETMBEGCgmSJomT8ixkARkWA2NvbTEdMBsGCgmSJomT
+8ixkARkWDWVsYXN0aWNzZWFyY2gxFDASBgoJkiaJk/IsZAEZFgR0ZXN0MRIwEAYK
+CZImiZPyLGQBGRYCYWQxHjAcBgNVBAMTFWFkLUVMQVNUSUNTRUFSQ0hBRC1DQTCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNNZsDJ+lhsE/pCIkNlq6/F
+xwv3PU2M+E1/SbWrLEtfbb1ATnn98DwxjpCj00wS0bt26/7zrhHKyX5LaxyS27ER
+8bKpLSO4qcVWzDIQnVNk2XfBrYS/Og+6Pi/Lw/ylt/vE++kHWIJBc4O6i+pPByOM
+oypM6bh71kTkpK8OTPqf+HiPp0qKhRah6XVtqTc+kOCOku2+wkELbCz8RNzF9ca6
+Uu3YxLi73pNdk0wDTmg6JVaUyVRpSkjJH4BAp9SVma6Rxy6tbh4e5P+8K8lY9ptM
+TBzTsDS1EhNK/92xULfQbGT814Z294pF3ARMEJ89N+aegS++kz7CqjciZ1+bA6EC
+AwEAAaNRME8wCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
+FIEKG0KdSVNknKcMZkbTlKo7N8MjMBAGCSsGAQQBgjcVAQQDAgEAMA0GCSqGSIb3
+DQEBCwUAA4IBAQBgbWBXPbEMTEsiVWzoxmTw1wJASBdPahx6CggutjGq3ASjby4p
+nVCTwE4xdDEVyFGmeslSp9+23XjBuaiqVPtYw8P8hnG269J0q4cOF/VXOccRLeOw
+HVDBv2a7xzgBSwc1KB50TLv07stcBmBYNu8anN6EwGksdgjb8IjRV6U3U+IvFNrI
+rGifuIc/iRZD4Clhnpxw8tCsgcrcmz9CU7CN5RxKVEpZ6ou6ZjHO8l8H0t9zWrSI
+PL+33iBGHNWlyU63N93XgJtxV1em1hHryLtTTtaVZJJ3R0OrLrUpG8SQ7zCUy62f
+YtImFPClUMXY03yH+4DAhflueRvY/D1AKL12
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_ca.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_ca.crt
new file mode 100644
index 00000000000..be5a6d02645
--- /dev/null
+++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_ca.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDmzCCAoOgAwIBAgIUdwsnIxjgSneHNVKT6JNCCsrQ3T0wDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMTgwMjE1MTc0ODQ2WhcNMjEwMjE0MTc0ODQ2WjA0MTIwMAYD
+VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKQAP/hdsWdu3Ck/Zteosity
+nmXJTCnkecBYSLtjgYh9rPDFppj9KdsZ7+5P9FvxLv/t4Yw81YI24TrHk0CnMrD/
+QBaXDiPGeT9b6T/gWWVm1zQj2/567gH2UaIkIffy7q09BI9ICXSKDBRXRMLgVR19
+iiJkwWb3b5TVvaQI4M8sEmJIHXei2/cfEKVR5hBprtzeKkvg6o9DXx+nDv2ZEUZ7
+it5pEN5AjD5t0S3ymtlUU5lqnr8er6/Qcrua2EXxE1HyPEkpN/Cwl7tF1ICMdguf
+vght5ql1/Pk43VmBMulI/6z5e+7GZ1+x79YA17gabtGJ+onB0zJxgDBj0tto7H8C
+AwEAAaOBpDCBoTAdBgNVHQ4EFgQUZo2Y3maL2NoxbbkwRZiC37k6QMEwbwYDVR0j
+BGgwZoAUZo2Y3maL2NoxbbkwRZiC37k6QMGhOKQ2MDQxMjAwBgNVBAMTKUVsYXN0
+aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBghR3CycjGOBKd4c1
+UpPok0IKytDdPTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBf
+mkc4bvUR5+We/2rRqCmP4LFnl/LxfbZ9/pUPRdcxuowuK7YfxN8i44VXGpJvLtec
+izhA8gvlj6GbYB/GNlHMogqEORbrMlu2o5Cev4HE/pcWpoqtVaDJqI5Hq4763EmJ
+p2dXGMmU04H4LtkcCEt3xQfLQ+QIP4Dl2yEsNd248BKSsscCGm9V3vgzFzbdgndo
+zUWv9hQCaEsKNtqvnkTqDy2uFjnf+xNoXFr/bI94gvD9HlZHnIC+g0TL5jjtSfCH
+gjeXhC2bBKFtlSt4ClIdZTXWievYs6YDRREfaOi4F0757A/gf+hT0fjZ+9WWnUeM
+UuvUnl71CNRnJ5JlNKBA
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_cert.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_cert.crt
new file mode 100644
index 00000000000..59ecbd22e8b
--- /dev/null
+++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/smb_cert.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDoDCCAoigAwIBAgIUMVGoHuyNTjTFaoRmqFELz75jzDEwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMTgwMjE1MTc0OTExWhcNMjEwMjE0MTc0OTExWjARMQ8wDQYD
+VQQDEwZzYW1iYTQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtGBwa
+n+7JN2vweSUsYh4zPmh8RPIE+nEVjK1lx/rADUBY7UVjfTYC+MVKKiezZe7gYCNT
+7JNKazPpgVI9e3ZFKw/UxomLqRuuvn5bTh+1tMs3afY5+GGzi7oPmEbBO3ceg0Hi
+rNSTDa1rfroZnRYK8uIeSZacQnAW90plITI7rBBt9jq+W9albFbDybfDgNv+yS/C
+rzIsofm4rbFC3SMRYfrT6HvwDhjOmmYKZci5x7tsn0T+3tSiR44Bw5/DgiN5kX3m
+/kl9qg1eoYWbCUy1dKmQlb4Nb4uNcxrIugLB3zjBkfhMZ0OHoveKh/lJASTWik9k
+xQ9rEYbpsRbuXpsHAgMBAAGjgcwwgckwHQYDVR0OBBYEFJOLa7UXKtLPibgKeFh7
+Kq1+rS0/MG8GA1UdIwRoMGaAFGaNmN5mi9jaMW25MEWYgt+5OkDBoTikNjA0MTIw
+MAYDVQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBD
+QYIUdwsnIxjgSneHNVKT6JNCCsrQ3T0wLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/
+AAABhxAAAAAAAAAAAAAAAAAAAAABMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQAD
+ggEBAEHqT1WHkcF8DuOgyIBx7wKcUVQ5H1qYYlJ1xgMGrKFFZLUzouLcON7oadEu
+HLIJ4Z3AKD3bqWpcls5XJ9MTECGR48tou67x9cXqTV7jR3Rh0H/VGwzwhR85vbpu
+o8ielOPL8XAQOfnAFESJii5sfCU4ZwLg+3evmGZdKfhU6rqQtLimgG/Gm96vOJne
+y0a/TZTWrfAarithkOHHXSSAhEI5SdW5SlZAytF4AmYqFvafwxe1+NyFwfCRy0Xl
+H40WgVsq+z84psU+WyORb3THX5rgB4au9nuMXOqFKAtrJSI/uApncYraaqU28rqB
+gYd8XrtjhKOLw+6viqAKu8l7/cs=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
index 9858a5cd118..a69445386d0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
@@ -44,12 +44,14 @@ public class WebhookHttpsIntegrationTests extends AbstractWatcherIntegrationTest
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        Path resource = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.jks");
+        Path keyPath = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.pem");
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.crt");
         return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put("xpack.http.ssl.keystore.path", resource.toString())
-                .put("xpack.http.ssl.keystore.password", "testnode")
-                .build();
+            .put(super.nodeSettings(nodeOrdinal))
+            .put("xpack.http.ssl.key", keyPath)
+            .put("xpack.http.ssl.certificate", certPath)
+            .put("xpack.http.ssl.keystore.password", "testnode")
+            .build();
     }
 
     @Before
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
index 10618b36e8a..03dcd794715 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
@@ -169,30 +169,31 @@ public class HttpClientTests extends ESTestCase {
     }
 
     public void testHttps() throws Exception {
-        Path resource = getDataPath("/org/elasticsearch/xpack/security/keystore/truststore-testnode-only.jks");
+        Path trustedCertPath = getDataPath("/org/elasticsearch/xpack/security/keystore/truststore-testnode-only.crt");
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.crt");
+        Path keyPath = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.pem");
         MockSecureSettings secureSettings = new MockSecureSettings();
         Settings settings;
         if (randomBoolean()) {
-            secureSettings.setString("xpack.http.ssl.truststore.secure_password", "truststore-testnode-only");
             settings = Settings.builder()
-                    .put("xpack.http.ssl.truststore.path", resource.toString())
-                    .setSecureSettings(secureSettings)
-                    .build();
+                .put("xpack.http.ssl.certificate_authorities", trustedCertPath)
+                .setSecureSettings(secureSettings)
+                .build();
         } else {
-            secureSettings.setString("xpack.ssl.truststore.secure_password", "truststore-testnode-only");
             settings = Settings.builder()
-                    .put("xpack.ssl.truststore.path", resource.toString())
-                    .setSecureSettings(secureSettings)
-                    .build();
+                .put("xpack.ssl.certificate_authorities", trustedCertPath)
+                .setSecureSettings(secureSettings)
+                .build();
         }
         try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) {
             secureSettings = new MockSecureSettings();
             // We can't use the client created above for the server since it is only a truststore
-            secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+            secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
             Settings settings2 = Settings.builder()
-                    .put("xpack.ssl.keystore.path", getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.jks"))
-                    .setSecureSettings(secureSettings)
-                    .build();
+                .put("xpack.ssl.key", keyPath)
+                .put("xpack.ssl.certificate", certPath)
+                .setSecureSettings(secureSettings)
+                .build();
 
             TestsSSLService sslService = new TestsSSLService(settings2, environment);
             testSslMockWebserver(client, sslService.sslContext(), false);
@@ -200,34 +201,40 @@ public class HttpClientTests extends ESTestCase {
     }
 
     public void testHttpsDisableHostnameVerification() throws Exception {
-        Path resource = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.jks");
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt");
+        Path keyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem");
         Settings settings;
         if (randomBoolean()) {
             MockSecureSettings secureSettings = new MockSecureSettings();
-            secureSettings.setString("xpack.http.ssl.truststore.secure_password", "testnode-no-subjaltname");
-            settings = Settings.builder()
-                    .put("xpack.http.ssl.truststore.path", resource.toString())
-                    .put("xpack.http.ssl.verification_mode", randomFrom(VerificationMode.NONE, VerificationMode.CERTIFICATE))
-                    .setSecureSettings(secureSettings)
-                    .build();
+            Settings.Builder builder = Settings.builder()
+                .put("xpack.http.ssl.certificate_authorities", certPath);
+            if (inFipsJvm()) {
+                //Can't use TrustAllConfig in FIPS mode
+                builder.put("xpack.http.ssl.verification_mode", VerificationMode.CERTIFICATE);
+            } else {
+                builder.put("xpack.http.ssl.verification_mode", randomFrom(VerificationMode.NONE, VerificationMode.CERTIFICATE));
+            }
+            settings = builder.build();
         } else {
-            MockSecureSettings secureSettings = new MockSecureSettings();
-            secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode-no-subjaltname");
-            settings = Settings.builder()
-                    .put("xpack.ssl.truststore.path", resource.toString())
-                    .put("xpack.ssl.verification_mode", randomFrom(VerificationMode.NONE, VerificationMode.CERTIFICATE))
-                    .setSecureSettings(secureSettings)
-                    .build();
+            Settings.Builder builder = Settings.builder()
+                .put("xpack.ssl.certificate_authorities", certPath);
+            if (inFipsJvm()) {
+                //Can't use TrustAllConfig in FIPS mode
+                builder.put("xpack.ssl.verification_mode", VerificationMode.CERTIFICATE);
+            } else {
+                builder.put("xpack.ssl.verification_mode", randomFrom(VerificationMode.NONE, VerificationMode.CERTIFICATE));
+            }
+            settings = builder.build();
         }
         try (HttpClient client = new HttpClient(settings, authRegistry, new SSLService(settings, environment))) {
             MockSecureSettings secureSettings = new MockSecureSettings();
             // We can't use the client created above for the server since it only defines a truststore
-            secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode-no-subjaltname");
+            secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode-no-subjaltname");
             Settings settings2 = Settings.builder()
-                    .put("xpack.ssl.keystore.path",
-                            getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.jks"))
-                    .setSecureSettings(secureSettings)
-                    .build();
+                .put("xpack.ssl.key", keyPath)
+                .put("xpack.ssl.certificate", certPath)
+                .setSecureSettings(secureSettings)
+                .build();
 
             TestsSSLService sslService = new TestsSSLService(settings2, environment);
             testSslMockWebserver(client, sslService.sslContext(), false);
@@ -235,13 +242,15 @@ public class HttpClientTests extends ESTestCase {
     }
 
     public void testHttpsClientAuth() throws Exception {
-        Path resource = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.jks");
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.crt");
+        Path keyPath = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.pem");
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
-                .put("xpack.ssl.keystore.path", resource.toString())
-                .setSecureSettings(secureSettings)
-                .build();
+            .put("xpack.ssl.key", keyPath)
+            .put("xpack.ssl.certificate", certPath)
+            .setSecureSettings(secureSettings)
+            .build();
 
         TestsSSLService sslService = new TestsSSLService(settings, environment);
         try (HttpClient client = new HttpClient(settings, authRegistry, sslService)) {
@@ -365,30 +374,31 @@ public class HttpClientTests extends ESTestCase {
     }
 
     public void testProxyCanHaveDifferentSchemeThanRequest() throws Exception {
+        Path trustedCertPath = getDataPath("/org/elasticsearch/xpack/security/keystore/truststore-testnode-only.crt");
+        Path certPath = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.crt");
+        Path keyPath = getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.pem");
         // this test fakes a proxy server that sends a response instead of forwarding it to the mock web server
         // on top of that the proxy request is HTTPS but the real request is HTTP only
         MockSecureSettings serverSecureSettings = new MockSecureSettings();
         // We can't use the client created above for the server since it is only a truststore
-        serverSecureSettings.setString("xpack.ssl.keystore.secure_password", "testnode");
+        serverSecureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
         Settings serverSettings = Settings.builder()
-                .put("xpack.ssl.keystore.path", getDataPath("/org/elasticsearch/xpack/security/keystore/testnode.jks"))
-                .setSecureSettings(serverSecureSettings)
-                .build();
+            .put("xpack.ssl.key", keyPath)
+            .put("xpack.ssl.certificate", certPath)
+            .setSecureSettings(serverSecureSettings)
+            .build();
         TestsSSLService sslService = new TestsSSLService(serverSettings, environment);
 
         try (MockWebServer proxyServer = new MockWebServer(sslService.sslContext(), false)) {
             proxyServer.enqueue(new MockResponse().setResponseCode(200).setBody("fullProxiedContent"));
             proxyServer.start();
 
-            Path resource = getDataPath("/org/elasticsearch/xpack/security/keystore/truststore-testnode-only.jks");
             MockSecureSettings secureSettings = new MockSecureSettings();
-            secureSettings.setString("xpack.http.ssl.truststore.secure_password", "truststore-testnode-only");
             Settings settings = Settings.builder()
                     .put(HttpSettings.PROXY_HOST.getKey(), "localhost")
                     .put(HttpSettings.PROXY_PORT.getKey(), proxyServer.getPort())
                     .put(HttpSettings.PROXY_SCHEME.getKey(), "https")
-                    .put("xpack.http.ssl.truststore.path", resource.toString())
-                    .setSecureSettings(secureSettings)
+                .put("xpack.http.ssl.certificate_authorities", trustedCertPath)
                     .build();
 
             HttpRequest.Builder requestBuilder = HttpRequest.builder("localhost", webServer.getPort())
diff --git a/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/testnode.crt b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/testnode.crt
new file mode 100644
index 00000000000..37e142afb23
--- /dev/null
+++ b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/testnode.crt
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDbTCCAlWgAwIBAgIJAJ+K5mGS3n/AMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV
+BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEgMB4GA1UEAxMXRWxhc3Rp
+Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTQxMjE2MTcwNDQ1WhcNMTgxMjE1MTcwNDQ1
+WjBIMQwwCgYDVQQKEwNvcmcxFjAUBgNVBAsTDWVsYXN0aWNzZWFyY2gxIDAeBgNV
+BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAzhpW7iwkm+Og+HP7U00nbmh0Hy9Z2Ldp5i8tJSlSQwTxCCvO
+rse6jwJQN98Dk1ApaSzimZrlKOotFyPV1L3fnOzJbTp1Yq/VsYP4zJkjWtID0qUf
+8Rg8bLhjKAG+ZlLuai5XZqnLkdmqvQeR61VhpXWFm0Om153tWmAiHL18ywY71gXN
+EnkeFo9OW4fDqkz6h7NJziYvU6URSKErZDEixk5GIPv9K9hiIfi0KQM6xaHp0d2w
+VCyFVC0OUdugz6untURzJVx4U3X1bQcv/o2BoUotWh/5h8o5eeiiv2OGZ1XlO+33
+1tweYI4wFjDwnAyHHRr/rk2ZIBiBYGaSzHnuhQIDAQABo1owWDAJBgNVHRMEAjAA
+MB0GA1UdDgQWBBTwGg2LF8+mzsvBBWxJKv6VXv3dMTAsBgNVHREEJTAjgglsb2Nh
+bGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAIwDQYJKoZIhvcNAQELBQADggEB
+ABP4ufLToJhcUselVxV9LPD5VGPEHGLdIFqsUEix7DMsiNpR76X6a8qNQbZpdbd6
++qPKqoaMgC7znX7qZtCqRbIXTWbudZPxFkcHdiWx3SiALMQYabeUGetClX3sCndU
+SUoV8f34i8dJxfNcqhLcsh4zpgxtmwsvs5OLMTBvm0Xo2zUFUjlmrt41pBrWEuq9
+nkObc/cr6Syiz3sy4pYVJO1/YwHaZgE/URqjVlari70DR3ES4YnIUnLQajKx2Q0/
+gXVgzjbe68KPOUGCz6GYiWq+d4tcWdHzLv1GsaqQ1MD9P21ArfrX4DpzgPDrO6MP
+9Ppq5DQGa2q4mz3kipd5RIs=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/testnode.pem b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/testnode.pem
new file mode 100644
index 00000000000..5de9c813fa8
--- /dev/null
+++ b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/testnode.pem
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAzhpW7iwkm+Og+HP7U00nbmh0Hy9Z2Ldp5i8tJSlSQwTxCCvO
+rse6jwJQN98Dk1ApaSzimZrlKOotFyPV1L3fnOzJbTp1Yq/VsYP4zJkjWtID0qUf
+8Rg8bLhjKAG+ZlLuai5XZqnLkdmqvQeR61VhpXWFm0Om153tWmAiHL18ywY71gXN
+EnkeFo9OW4fDqkz6h7NJziYvU6URSKErZDEixk5GIPv9K9hiIfi0KQM6xaHp0d2w
+VCyFVC0OUdugz6untURzJVx4U3X1bQcv/o2BoUotWh/5h8o5eeiiv2OGZ1XlO+33
+1tweYI4wFjDwnAyHHRr/rk2ZIBiBYGaSzHnuhQIDAQABAoIBAQCWgv3A6VPC1DUV
+u/1qFAobwwQqUfYXIbqgcwtQ/CAq+UzcXsGNOiavkUzrwF1oEz5qpHDHJCr9+iX7
+pBvgRNksTG+86NgYvbgc7vee0qbTCFPqXNQ6ySw3aWBgMwXMy/t4Z2dEffNAC+l4
+KjMR3UR2BKERhhItnBNd0J6Yxnh/+jg1Uf5fVMEh1/WOGLYCJnnn5oEHNKeon6XR
+dobIn2QjD/PB8ZX7UubrSXmyezU0e9h3ARoI3oPMV6f8XQSa5K/KRrk8FUkVQ4vI
+5+YAMjtY/K2I8xAEoPyprD/ILAVN+3E47J0K14EfKNTajSzQFVJhaaCvs7btxScA
+Sx/zRsvxAoGBAP5KMH6vamdnBlZTPT2jtrsmzjyC0Z+9lbNokzRmVribps+DFdAW
+YsGCbfApcbOYmpdLSeccFTA+uT5IbQ8hwBbWn/HKm+y8EDAPklf5tL0+w7pCZ4kU
+50pKk6cjSTv/CDjO+hy4KIz2H/zXivXEV+4FtFKOZ3qUVg7m+1c/u5lDAoGBAM99
+L8/S9jwCkOjv+TKhmK+2/S5tVy1fdjlurTu9nI46CYa9MaOndZKY6EJ9ekBLIHUQ
+h1QAsdPRHgkObuKDUHmpLr7qmoTRE7vtWC3sHK382j5CBEK00p+09wFHA03Bf40f
+Jdjlzqe9F9jO6LH2RL/TECQDe7RJaTOQJrNlVtiXAoGBAOUUsNtv68t7ZJogIuuE
+sPmo2+Jnd7EQeexGKVbrWvS0RHJtBRmRESaC+ceBjozczWe+y7UH946e8wLI/HbF
+UOdCMpUAkbeTNIIXhR78NXbHNEx3xg4YZsTmql3HzBHgjueejnOQ8/cJQ4fkJauC
+VjR3rxswbshfGagTLhpLsBVBAoGBAMBf5mN+ynRLQMXoMRlDgIhyVf2kvO5BkyCe
+wBkirTOlFc4KPirpCXZ5NObo5d8UiKxhPcehuT6VpY5qBl8XtxaFyOSUKd24594W
+qeox/0lFpaeRl9etRZdztoxFpgeCv1s9pN6b+2XESYboGBFgLs/XxiBN5nT6l4KK
+RYeRDttTAoGAMoAreVa/i1l5YChhyskBTt+nePHGomsXC9tv7mZFLOrPQ+CLy5Xd
+4PQqqYjRaJT/aP3N/q0NcauSKxYKmgnhInXpaasSVzGrM60DQLVw+SXfTiwXN0dH
+V/bq2ybdSxEh2xQoyrfpiFDkCEecY0nYCL1Ff7UYY6g8P/Qj8DBiZGI=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/truststore-testnode-only.crt b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/truststore-testnode-only.crt
new file mode 100644
index 00000000000..37e142afb23
--- /dev/null
+++ b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/keystore/truststore-testnode-only.crt
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDbTCCAlWgAwIBAgIJAJ+K5mGS3n/AMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV
+BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEgMB4GA1UEAxMXRWxhc3Rp
+Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTQxMjE2MTcwNDQ1WhcNMTgxMjE1MTcwNDQ1
+WjBIMQwwCgYDVQQKEwNvcmcxFjAUBgNVBAsTDWVsYXN0aWNzZWFyY2gxIDAeBgNV
+BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAzhpW7iwkm+Og+HP7U00nbmh0Hy9Z2Ldp5i8tJSlSQwTxCCvO
+rse6jwJQN98Dk1ApaSzimZrlKOotFyPV1L3fnOzJbTp1Yq/VsYP4zJkjWtID0qUf
+8Rg8bLhjKAG+ZlLuai5XZqnLkdmqvQeR61VhpXWFm0Om153tWmAiHL18ywY71gXN
+EnkeFo9OW4fDqkz6h7NJziYvU6URSKErZDEixk5GIPv9K9hiIfi0KQM6xaHp0d2w
+VCyFVC0OUdugz6untURzJVx4U3X1bQcv/o2BoUotWh/5h8o5eeiiv2OGZ1XlO+33
+1tweYI4wFjDwnAyHHRr/rk2ZIBiBYGaSzHnuhQIDAQABo1owWDAJBgNVHRMEAjAA
+MB0GA1UdDgQWBBTwGg2LF8+mzsvBBWxJKv6VXv3dMTAsBgNVHREEJTAjgglsb2Nh
+bGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAIwDQYJKoZIhvcNAQELBQADggEB
+ABP4ufLToJhcUselVxV9LPD5VGPEHGLdIFqsUEix7DMsiNpR76X6a8qNQbZpdbd6
++qPKqoaMgC7znX7qZtCqRbIXTWbudZPxFkcHdiWx3SiALMQYabeUGetClX3sCndU
+SUoV8f34i8dJxfNcqhLcsh4zpgxtmwsvs5OLMTBvm0Xo2zUFUjlmrt41pBrWEuq9
+nkObc/cr6Syiz3sy4pYVJO1/YwHaZgE/URqjVlari70DR3ES4YnIUnLQajKx2Q0/
+gXVgzjbe68KPOUGCz6GYiWq+d4tcWdHzLv1GsaqQ1MD9P21ArfrX4DpzgPDrO6MP
+9Ppq5DQGa2q4mz3kipd5RIs=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt
new file mode 100644
index 00000000000..ced9d81d96f
--- /dev/null
+++ b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDTTCCAjWgAwIBAgIJALL7dwEsWamvMA0GCSqGSIb3DQEBCwUAME8xDDAKBgNV
+BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEnMCUGA1UEAxMeRWxhc3Rp
+Y3NlYXJjaCBUZXN0IE5vZGUgTm8gU0FOMB4XDTE0MTIxNjE5NTcyNloXDTE4MTIx
+NTE5NTcyNlowTzEMMAoGA1UEChMDb3JnMRYwFAYDVQQLEw1lbGFzdGljc2VhcmNo
+MScwJQYDVQQDEx5FbGFzdGljc2VhcmNoIFRlc3QgTm9kZSBObyBTQU4wggEiMA0G
+CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCkIGS7A/V6TesR34ajMyNYL3tB1OjW
+Raq4KtF8FfW1H6nHGrWa/qXjZWPirczy1k2n6ZL7YOCcv/YeY8xAqC9mGQxvEuqo
+EaqXq2cjRdAs/7zqzRkdPPi3Jw/p/RHrDfOAzOsMnBGc0G2Hrsj//aP44vp85pek
+fM3t2kNAYZWYCzXUqWAIUoxBDK4DcQdsN8H4KTMIwQEEiRtcKnL/b8QGKsyGLfLq
+36ZABHZ4kY2SmcP3bWxZtbFN4hamdwoAtYe+lS0/ee8/fOTLyZ3Ey+X6EEmGO1lk
+WR4XLli15k1L2HBzWGG7zwxVEC5r2h3Sx1njYh/Jq3khIdSvDbiMmM+VAgMBAAGj
+LDAqMAkGA1UdEwQCMAAwHQYDVR0OBBYEFGm8wrYF9mJweJ1vloDw19e0PUuIMA0G
+CSqGSIb3DQEBCwUAA4IBAQBbEZ73weDphNIcmvN25v6NIfjBebqgm0/2grDFwmZe
+Z1DibzRoVfoQ7WeUqbPS7SHUQ+KzIN1GdfHXhW9r6mmLbtzPv90Q/8zBcNv5HNZZ
+YK+T2r9hoAWEY6nB1fiOJ4udkFMYfAi6LiSxave4IPWp/WIqd0IWtPtkPl+MmG41
+TfRom8TnO+o+VsjgDkY5Q1JDsNQKy1BrtxzIZyz7d1zYKTQ+HXZ4yeYJoVoc3k4y
+6w9eX2zAUZ6Z3d4an6CLr6Hew9Dj2VX1vqCj1a5/VvHZVyVxyh4hg8sHYm7tZOJX
+wN3B5GcKwbbFjaMVBLaMlP62OdGg7tCh61evWm+l06S0
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem
new file mode 100644
index 00000000000..b0f7a585d7f
--- /dev/null
+++ b/x-pack/plugin/watcher/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.pem
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-192-CBC,2F36F79E75ACA7803AF1BC1B70C2360C
+
+d4/f7dnpPW9DfhGXuGDx7r56BjQ64iNcsQdrC1CTZB363iAvBBeHaoJFaWpQOUmT
+WCBdM6YmRuMi6z4sMtm8Vre3i+Mu2Jti0LTDXqH8Y6FMpptxAYoFb9hkM0OvUGDJ
+ASLGTa1XKyCzZWOe2IGLtWIS8zkGv/fSXHqDJV8ddq8uLHGZXBxAye5oJUD0654T
+DasIllbiTlBEYlEJ10OyIAspS47n4LWVE5ggyGdiyIgh5s3lAMPO3+yenDU3/dzj
+YF2zHKnfe0Nx2lUQVJRYknQvFCx0WTGH9gNl40FovoOMHyRjEuOHD9PFNvL/TCh6
+9h0jJPWu/SsIiYaIqR0pDgqWdHXHA5Ea3s2+w0YHbv7DqkGXWZdceLUdZtcXCBJk
+P8QL9IWa7VE50SS6wV5uXX9tV5lHzMFsNGkwtGcR1UMU5cXYqckFXgoBqDN0fyWe
+V5iEknSJ4Na+MHv75rvRat0kv6upu9i5NSpYTc5jLHdWZWsYMZ/ZMiMoLBP+KAPT
+DQ3eyph/84BU3DePaQF3Rsp0ZvPxqQ361Zwc4zC5CKbHur1QX8WAY5XlBMfuBpkf
+CKr5wgwF+ZpS7zsfUpMPPe9Y1E8TWnhx/DtCVLEslBpr2u/rMaxPp6ev9/Wry7N+
+UFBOwodipBhlfSvLqjc511L+bXRzqXiINuW0eSKUQv0J/G0Ur894kJJ6feDYlskj
+JiZjOgOyyKhB+K9AXmkfRdvWUJeweL8pgDuYSyQdQ0zoUCZALEoYK2cBWzti/wep
+QPFD5oz8076aXNHKMHLsRmSFuEZb6IN0PtUNVf958EbrtABNIuoufKlKtJsEnUyK
+VHIEUxExEgyECiozKnxvhr7RQ9nTQXhNdgtec6jJblYnla/+OWAfHdxtHhBjp5AX
+WvLyUhmgrmLNdKd1KSzcXynBHgh0hi0HJXYx31FilwbxsdhwN1LwN/Do4T4qGkUr
+InrQC3ZHRuh0yAOPrwRFEWbW5m/PzVP/xYVgFtVWP7w38ftZbaBy5xPmtswn+PH+
+cIMt1Y9PaAlhLNpW/Vfn503T9M+05vu73vbU1xgu/B1kePOqE/WO0cOZl0KdaMmT
+wAQBKuI7qTACH+/8g3Uir1YSChLthH+1Gs6h686EP6ZydwXq9GYXXkNmJNJJsnmU
+RDjoT0F4XBKvcQdX3EeQYs3Af2yZWFDC59c1Ews2dqMK7vy2tYITbx2yn30DBDAl
+xvjp2izzmAgQJEG/RqCYsUHCCEv7wz3tpsSOkFem9IHZpR2h8Rqy88GH9qYOkgwo
++fKSmIgC4RLQXsHuh7RRuyNc2FaWDgRgSxs5V4f9xOSU/ZbUftYWnwEyCwbu3RJp
+CIXQFZhzU2t5l1Eh+x40rwpEJDXBEwmOIUO3x1oOqGZPPEQ674uMal5TRjvdOVGD
+h665Fpo5Xu9EQwQZHYddeRl/7yw8F6LCxBLgHlngKRHHGDUHlTscLfYRqNh+x3jT
+3S8dfaGzlnwdQEx32gyLAV0/nsFnzh1AknFMT8jesIYF7PLiAi67PNyNwRCc7TFp
+jpKvzkDRVP72bivTmCyP5aKR0Q2oIrAw51MMinT6R2VaoR7COjoVbqYsRLwkxu+p
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/qa/openldap-tests/build.gradle b/x-pack/qa/openldap-tests/build.gradle
index 24976ab6113..5c0399a1d9f 100644
--- a/x-pack/qa/openldap-tests/build.gradle
+++ b/x-pack/qa/openldap-tests/build.gradle
@@ -17,6 +17,7 @@ task openLdapFixture {
 String outputDir = "${project.buildDir}/generated-resources/${project.name}"
 task copyIdpTrust(type: Copy) {
     from idpFixtureProject.file('src/main/resources/certs/idptrust.jks');
+    from idpFixtureProject.file('src/main/resources/certs/ca.crt');
     into outputDir
 }
 if (project.rootProject.vagrantSupported) {
diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java
index 1c9d93873a4..c7a92dccab8 100644
--- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java
+++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java
@@ -34,7 +34,6 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
 
-import static org.elasticsearch.test.OpenLdapTests.LDAPTRUST_PATH;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -45,22 +44,20 @@ public class OpenLdapUserSearchSessionFactoryTests extends ESTestCase {
 
     private Settings globalSettings;
     private ThreadPool threadPool;
-    private MockSecureSettings globalSecureSettings;
+    private static final String LDAPCACERT_PATH = "/ca.crt";
 
     @Before
     public void init() throws Exception {
-        Path keystore = getDataPath(LDAPTRUST_PATH);
+        Path caPath = getDataPath(LDAPCACERT_PATH);
         /*
          * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext.
          * If we re-use a SSLContext, previously connected sessions can get re-established which breaks hostname
          * verification tests since a re-established connection does not perform hostname verification.
          */
-        globalSecureSettings = newSecureSettings("xpack.ssl.truststore.secure_password", "changeit");
         globalSettings = Settings.builder()
-                .put("path.home", createTempDir())
-                .put("xpack.ssl.truststore.path", keystore)
-                .setSecureSettings(globalSecureSettings)
-                .build();
+            .put("path.home", createTempDir())
+            .put("xpack.ssl.certificate_authorities", caPath)
+            .build();
         threadPool = new TestThreadPool("LdapUserSearchSessionFactoryTests");
     }
 
@@ -94,7 +91,6 @@ public class OpenLdapUserSearchSessionFactoryTests extends ESTestCase {
                 .put(globalSettings, false);
         builder.put(Settings.builder().put(config.settings(), false).normalizePrefix("xpack.security.authc.realms.oldap-test.").build());
         final MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.merge(globalSecureSettings);
         if (useSecureBindPassword) {
             secureSettings.setString("xpack.security.authc.realms.oldap-test.secure_bind_password", OpenLdapTests.PASSWORD);
         }
diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java
index f277f5d84b3..b55431dee1b 100644
--- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java
+++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java
@@ -177,4 +177,4 @@ public class SearchGroupsResolverTests extends GroupsResolverTestCase {
     protected String trustPath() {
         return "/idptrust.jks";
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateToolTests.java b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateToolTests.java
index e11b62642eb..dde0b7645df 100644
--- a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateToolTests.java
+++ b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateToolTests.java
@@ -38,6 +38,7 @@ import org.elasticsearch.xpack.core.ssl.CertificateGenerateTool.CertificateInfor
 import org.elasticsearch.xpack.core.ssl.CertificateGenerateTool.Name;
 import org.hamcrest.Matchers;
 import org.junit.After;
+import org.junit.BeforeClass;
 
 import javax.security.auth.x500.X500Principal;
 
@@ -96,6 +97,11 @@ public class CertificateGenerateToolTests extends ESTestCase {
         return tempDir;
     }
 
+    @BeforeClass
+    public static void checkFipsJvm() {
+        assumeFalse("Can't run in a FIPS JVM, depends on Non FIPS BouncyCastle", inFipsJvm());
+    }
+
     @After
     public void tearDown() throws Exception {
         IOUtils.close(jimfs);
diff --git a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateToolTests.java b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateToolTests.java
index 795dd074a80..706d5dbab5f 100644
--- a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateToolTests.java
+++ b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateToolTests.java
@@ -47,6 +47,7 @@ import org.elasticsearch.xpack.core.ssl.CertificateTool.GenerateCertificateComma
 import org.elasticsearch.xpack.core.ssl.CertificateTool.Name;
 import org.hamcrest.Matchers;
 import org.junit.After;
+import org.junit.BeforeClass;
 
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.TrustManagerFactory;
@@ -108,6 +109,11 @@ public class CertificateToolTests extends ESTestCase {
         return tempDir;
     }
 
+    @BeforeClass
+    public static void chechFipsJvm() {
+        assumeFalse("Can't run in a FIPS JVM, depends on Non FIPS BouncyCastle", inFipsJvm());
+    }
+
     @After
     public void tearDown() throws Exception {
         IOUtils.close(jimfs);
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java
index 98594917129..32452a609e2 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java
@@ -40,7 +40,7 @@ public class ADLdapUserSearchSessionFactoryTests extends AbstractActiveDirectory
 
     @Before
     public void init() throws Exception {
-        Path keystore = getDataPath("support/ADtrust.jks");
+        Path certPath = getDataPath("support/smb_ca.crt");
         Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         /*
          * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext.
@@ -49,10 +49,9 @@ public class ADLdapUserSearchSessionFactoryTests extends AbstractActiveDirectory
          */
 
         globalSettings = Settings.builder()
-                .put("path.home", createTempDir())
-                .put("xpack.ssl.truststore.path", keystore)
-                .setSecureSettings(newSecureSettings("xpack.ssl.truststore.secure_password", "changeit"))
-                .build();
+            .put("path.home", createTempDir())
+            .put("xpack.ssl.certificate_authorities", certPath)
+            .build();
         sslService = new SSLService(globalSettings, env);
         threadPool = new TestThreadPool("ADLdapUserSearchSessionFactoryTests");
     }
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractActiveDirectoryTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractActiveDirectoryTestCase.java
index 7ef1bd674a3..829e87c849d 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractActiveDirectoryTestCase.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractActiveDirectoryTestCase.java
@@ -23,9 +23,16 @@ import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.VerificationMode;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.List;
 
 public abstract class AbstractActiveDirectoryTestCase extends ESTestCase {
 
@@ -48,11 +55,25 @@ public abstract class AbstractActiveDirectoryTestCase extends ESTestCase {
     protected SSLService sslService;
     protected Settings globalSettings;
     protected boolean useGlobalSSL;
+    protected List<String> certificatePaths;
 
     @Before
     public void initializeSslSocketFactory() throws Exception {
         useGlobalSSL = randomBoolean();
-        Path truststore = getDataPath("../ldap/support/ADtrust.jks");
+        // We use certificates in PEM format and `ssl.certificate_authorities` instead of ssl.trustore
+        // so that these tests can also run in a FIPS JVM where JKS keystores can't be used.
+        certificatePaths = new ArrayList<>();
+        Files.walkFileTree(getDataPath
+            ("../ldap/support"), new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                String fileName = file.getFileName().toString();
+                if (fileName.endsWith(".crt")) {
+                    certificatePaths.add(getDataPath("../ldap/support/" + fileName).toString());
+                }
+                return FileVisitResult.CONTINUE;
+            }
+        });
         /*
          * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext.
          * If we re-use a SSLContext, previously connected sessions can get re-established which breaks hostname
@@ -60,20 +81,16 @@ public abstract class AbstractActiveDirectoryTestCase extends ESTestCase {
          */
         Settings.Builder builder = Settings.builder().put("path.home", createTempDir());
         if (useGlobalSSL) {
-            builder.put("xpack.ssl.truststore.path", truststore)
-                    .put("xpack.ssl.truststore.password", "changeit");
+            builder.putList("xpack.ssl.certificate_authorities", certificatePaths);
 
             // fake realm to load config with certificate verification mode
-            builder.put("xpack.security.authc.realms.bar.ssl.truststore.path", truststore);
-            builder.put("xpack.security.authc.realms.bar.ssl.truststore.password", "changeit");
+            builder.putList("xpack.security.authc.realms.bar.ssl.certificate_authorities", certificatePaths);
             builder.put("xpack.security.authc.realms.bar.ssl.verification_mode", VerificationMode.CERTIFICATE);
         } else {
             // fake realms so ssl will get loaded
-            builder.put("xpack.security.authc.realms.foo.ssl.truststore.path", truststore);
-            builder.put("xpack.security.authc.realms.foo.ssl.truststore.password", "changeit");
+            builder.putList("xpack.security.authc.realms.foo.ssl.certificate_authorities", certificatePaths);
             builder.put("xpack.security.authc.realms.foo.ssl.verification_mode", VerificationMode.FULL);
-            builder.put("xpack.security.authc.realms.bar.ssl.truststore.path", truststore);
-            builder.put("xpack.security.authc.realms.bar.ssl.truststore.password", "changeit");
+            builder.putList("xpack.security.authc.realms.bar.ssl.certificate_authorities", certificatePaths);
             builder.put("xpack.security.authc.realms.bar.ssl.verification_mode", VerificationMode.CERTIFICATE);
         }
         globalSettings = builder.build();
@@ -99,8 +116,7 @@ public abstract class AbstractActiveDirectoryTestCase extends ESTestCase {
             builder.put(SessionFactorySettings.HOSTNAME_VERIFICATION_SETTING, hostnameVerification);
         }
         if (useGlobalSSL == false) {
-            builder.put("ssl.truststore.path", getDataPath("../ldap/support/ADtrust.jks"))
-                    .put("ssl.truststore.password", "changeit");
+            builder.putList("ssl.certificate_authorities", certificatePaths);
         }
         return builder.build();
     }
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
index 11da59e44d6..d35e7ab0b0c 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
@@ -21,7 +21,6 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
 import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings;
@@ -100,7 +99,8 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
             )
     };
 
-    protected static final String TESTNODE_KEYSTORE = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks";
+    protected static final String TESTNODE_KEY = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem";
+    protected static final String TESTNODE_CERT = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt";
     protected static RealmConfig realmConfig;
     protected static List<RoleMappingEntry> roleMappings;
     protected static boolean useGlobalSSL;
@@ -122,7 +122,8 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         final RealmConfig realm = AbstractAdLdapRealmTestCase.realmConfig;
-        Path store = getDataPath(TESTNODE_KEYSTORE);
+        final Path nodeCert = getDataPath(TESTNODE_CERT);
+        final Path nodeKey = getDataPath(TESTNODE_KEY);
         Settings.Builder builder = Settings.builder();
         // don't use filter since it returns a prefixed secure setting instead of mock!
         Settings settingsToAdd = super.nodeSettings(nodeOrdinal);
@@ -156,14 +157,15 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
                 }
             }
         }
-        addSslSettingsForStore(builder, store, "testnode");
-        builder.put(buildRealmSettings(realm, roleMappings, store));
+        addSslSettingsForKeyPair(builder, nodeKey, "testnode", nodeCert, getNodeTrustedCertificates());
+        builder.put(buildRealmSettings(realm, roleMappings, getNodeTrustedCertificates()));
         return builder.build();
     }
 
-    protected Settings buildRealmSettings(RealmConfig realm, List<RoleMappingEntry> roleMappingEntries, Path store) {
+    protected Settings buildRealmSettings(RealmConfig realm, List<RoleMappingEntry> roleMappingEntries, List<String>
+        certificateAuthorities) {
         Settings.Builder builder = Settings.builder();
-        builder.put(realm.buildSettings(store, "testnode"));
+        builder.put(realm.buildSettings(certificateAuthorities));
         configureFileRoleMappings(builder, roleMappingEntries);
         return builder.build();
     }
@@ -216,10 +218,11 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
     @Override
     protected Settings transportClientSettings() {
         if (useGlobalSSL) {
-            Path store = getDataPath(TESTNODE_KEYSTORE);
+            Path key = getDataPath(TESTNODE_KEY);
+            Path cert = getDataPath(TESTNODE_CERT);
             Settings.Builder builder = Settings.builder()
                     .put(super.transportClientSettings().filter((s) -> s.startsWith("xpack.ssl.") == false));
-            addSslSettingsForStore(builder, store, "testnode");
+            addSslSettingsForKeyPair(builder, key, "testnode", cert, getNodeTrustedCertificates());
             return builder.build();
         } else {
             return super.transportClientSettings();
@@ -304,14 +307,35 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
         return UsernamePasswordToken.basicAuthHeaderValue(username, new SecureString(password.toCharArray()));
     }
 
-    private void addSslSettingsForStore(Settings.Builder builder, Path store, String password) {
-        SecuritySettingsSource.addSecureSettings(builder, secureSettings -> {
-                secureSettings.setString("xpack.ssl.keystore.secure_password", password);
-                secureSettings.setString("xpack.ssl.truststore.secure_password", password);
-        });
-        builder.put("xpack.ssl.keystore.path", store)
-               .put("xpack.ssl.verification_mode", "certificate")
-               .put("xpack.ssl.truststore.path", store);
+    private void addSslSettingsForKeyPair(Settings.Builder builder, Path key, String keyPassphrase, Path cert,
+                                          List<String> certificateAuthorities) {
+        builder.put("xpack.ssl.key", key)
+            .put("xpack.ssl.key_passphrase", keyPassphrase)
+            .put("xpack.ssl.verification_mode", "certificate")
+            .put("xpack.ssl.certificate", cert)
+            .putList("xpack.ssl.certificate_authorities", certificateAuthorities);
+    }
+
+    /**
+     * Collects all the certificates that are normally trusted by the node ( contained in testnode.jks )
+     *
+     * @return
+     */
+    List<String> getNodeTrustedCertificates() {
+        Path testnodeCert =
+            getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
+        Path testnodeClientProfileCert =
+            getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.crt");
+        Path activedirCert =
+            getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/active-directory-ca.crt");
+        Path testclientCert =
+            getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt");
+        Path openldapCert =
+            getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openldap.crt");
+        Path samba4Cert =
+            getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt");
+        return Arrays.asList(testnodeCert.toString(), testnodeClientProfileCert.toString(), activedirCert.toString(), testclientCert
+            .toString(), openldapCert.toString(), samba4Cert.toString());
     }
 
     static class RoleMappingEntry {
@@ -429,19 +453,19 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
             this.mapGroupsAsRoles = randomBoolean();
         }
 
-        public Settings buildSettings(Path store, String password) {
-            return buildSettings(store, password, 1);
+        public Settings buildSettings(List<String> certificateAuthorities) {
+            return buildSettings(certificateAuthorities, 1);
         }
 
-        protected Settings buildSettings(Path store, String password, int order) {
+
+        protected Settings buildSettings(List<String> certificateAuthorities, int order) {
             Settings.Builder builder = Settings.builder()
-                    .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".order", order)
-                    .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".hostname_verification", false)
-                    .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".unmapped_groups_as_roles", mapGroupsAsRoles)
-                    .put(this.settings);
+                .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".order", order)
+                .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".hostname_verification", false)
+                .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".unmapped_groups_as_roles", mapGroupsAsRoles)
+                .put(this.settings);
             if (useGlobalSSL == false) {
-                builder.put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".ssl.truststore.path", store)
-                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".ssl.truststore.password", password);
+                builder.putList(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".ssl.certificate_authorities", certificateAuthorities);
             }
 
             return builder.build();
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java
index fb7ea6c5dd7..330ec6b9a75 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java
@@ -132,4 +132,4 @@ public class ActiveDirectoryGroupsResolverTests extends GroupsResolverTestCase {
     protected String trustPath() {
         return "/org/elasticsearch/xpack/security/authc/ldap/support/ADtrust.jks";
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
index 8f85c250f48..614d6659f2d 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
@@ -280,10 +280,9 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
                 .build();
         if (useGlobalSSL == false) {
             settings = Settings.builder()
-                    .put(settings)
-                    .put("ssl.truststore.path", getDataPath("../ldap/support/ADtrust.jks"))
-                    .put("ssl.truststore.password", "changeit")
-                    .build();
+                .put(settings)
+                .putList("ssl.certificate_authorities", certificatePaths)
+                .build();
         }
         RealmConfig config = new RealmConfig("ad-as-ldap-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
                 new ThreadContext(globalSettings));
@@ -317,10 +316,9 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
                 ignoreReferralErrors);
         if (useGlobalSSL == false) {
             settings = Settings.builder()
-                    .put(settings)
-                    .put("ssl.truststore.path", getDataPath("../ldap/support/ADtrust.jks"))
-                    .put("ssl.truststore.password", "changeit")
-                    .build();
+                .put(settings)
+                .putList("ssl.certificate_authorities", certificatePaths)
+                .build();
         }
         RealmConfig config = new RealmConfig("ad-as-ldap-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
                 new ThreadContext(globalSettings));
@@ -346,10 +344,9 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
         Settings settings = LdapTestCase.buildLdapSettings(new String[] { AD_LDAP_URL }, userTemplate, false);
         if (useGlobalSSL == false) {
             settings = Settings.builder()
-                    .put(settings)
-                    .put("ssl.truststore.path", getDataPath("../ldap/support/ADtrust.jks"))
-                    .put("ssl.truststore.password", "changeit")
-                    .build();
+                .put(settings)
+                .putList("ssl.certificate_authorities", certificatePaths)
+                .build();
         }
         RealmConfig config = new RealmConfig("ad-as-ldap-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
                 new ThreadContext(globalSettings));
@@ -408,8 +405,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
         }
 
         if (useGlobalSSL == false) {
-            builder.put("ssl.truststore.path", getDataPath("../ldap/support/ADtrust.jks"))
-                    .put("ssl.truststore.password", "changeit");
+            builder.putList("ssl.certificate_authorities", certificatePaths);
         }
 
         if (useBindUser) {
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/MultipleAdRealmIT.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/MultipleAdRealmIT.java
index c4e07a846fd..3d1cdb202d1 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/MultipleAdRealmIT.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/MultipleAdRealmIT.java
@@ -10,7 +10,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -46,9 +45,9 @@ public class MultipleAdRealmIT extends AbstractAdLdapRealmTestCase {
         Settings.Builder builder = Settings.builder();
         builder.put(super.nodeSettings(nodeOrdinal));
 
-        Path store = getDataPath(TESTNODE_KEYSTORE);
         final List<RoleMappingEntry> secondaryRoleMappings = secondaryRealmConfig.selectRoleMappings(() -> true);
-        final Settings secondarySettings = super.buildRealmSettings(secondaryRealmConfig, secondaryRoleMappings, store);
+        final Settings secondarySettings = super.buildRealmSettings(secondaryRealmConfig, secondaryRoleMappings,
+            getNodeTrustedCertificates());
         secondarySettings.keySet().forEach(name -> {
             String newName = name.replace(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL, XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + "2");
             builder.copy(newName, name, secondarySettings);

From 049966a829918a61cf3ac181e709183c9ef686e8 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Tue, 17 Jul 2018 11:26:28 +0200
Subject: [PATCH 056/260] Check that client methods match API defined in the
 REST spec (#31825)

We have been encountering name mismatches between API defined in our
REST spec and method names that have been added to the high-level REST
client. We should check this automatically to prevent furher mismatches,
and correct all the current ones.

This commit adds a test for this and corrects the issues found by it.
---
 .../gradle/test/RestIntegTestTask.groovy      |   3 +-
 client/rest-high-level/build.gradle           |  10 +-
 .../elasticsearch/client/IndicesClient.java   |  35 +++-
 .../elasticsearch/client/IngestClient.java    |   8 +-
 .../client/RestHighLevelClient.java           | 102 ++++++++++-
 .../elasticsearch/client/SnapshotClient.java  |   6 +-
 .../elasticsearch/client/BulkProcessorIT.java |  10 +-
 .../client/BulkProcessorRetryIT.java          |   2 +-
 .../java/org/elasticsearch/client/CrudIT.java |   4 +-
 .../CustomRestHighLevelClientTests.java       |   2 +-
 .../elasticsearch/client/IndicesClientIT.java |   6 +-
 .../elasticsearch/client/IngestClientIT.java  |   2 +-
 .../client/RestHighLevelClientTests.java      | 161 +++++++++++++++++-
 .../org/elasticsearch/client/SearchIT.java    |  24 +--
 .../org/elasticsearch/client/SnapshotIT.java  |  14 +-
 .../documentation/CRUDDocumentationIT.java    |  14 +-
 .../IndicesClientDocumentationIT.java         |  10 +-
 .../IngestClientDocumentationIT.java          |   4 +-
 .../documentation/SearchDocumentationIT.java  |  20 +--
 .../SnapshotClientDocumentationIT.java        |   4 +-
 ...rossClusterSearchUnavailableClusterIT.java |   4 +-
 21 files changed, 359 insertions(+), 86 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
index f2e6dc8e561..d2101c48aab 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
@@ -24,7 +24,6 @@ import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.DefaultTask
 import org.gradle.api.Project
 import org.gradle.api.Task
-import org.gradle.api.Transformer
 import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.provider.Property
@@ -217,7 +216,7 @@ public class RestIntegTestTask extends DefaultTask {
      * @param project The project to add the copy task to
      * @param includePackagedTests true if the packaged tests should be copied, false otherwise
      */
-    private static Task createCopyRestSpecTask(Project project, Provider<Boolean> includePackagedTests) {
+    static Task createCopyRestSpecTask(Project project, Provider<Boolean> includePackagedTests) {
         project.configurations {
             restSpec
         }
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index 451452759f5..2fed806e98c 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -18,8 +18,8 @@
  */
 
 import org.elasticsearch.gradle.precommit.PrecommitTasks
-import org.gradle.api.XmlProvider
-import org.gradle.api.publish.maven.MavenPublication
+import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.gradle.api.internal.provider.Providers
 
 buildscript {
   repositories {
@@ -41,6 +41,10 @@ apply plugin: 'com.github.johnrengelman.shadow'
 group = 'org.elasticsearch.client'
 archivesBaseName = 'elasticsearch-rest-high-level-client'
 
+//we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions)
+Task copyRestSpec = RestIntegTestTask.createCopyRestSpecTask(project, Providers.FALSE)
+test.dependsOn(copyRestSpec)
+
 publishing {
   publications {
     nebula(MavenPublication) {
@@ -102,6 +106,8 @@ dependencies {
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+  //this is needed to make RestHighLevelClientTests#testApiNamingConventions work from IDEs
+  testCompile "org.elasticsearch:rest-api-spec:${version}"
 }
 
 dependencyLicenses {
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
index 2944b49bf18..250bbd520da 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
@@ -174,7 +174,7 @@ public final class IndicesClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public GetMappingsResponse getMappings(GetMappingsRequest getMappingsRequest, RequestOptions options) throws IOException {
+    public GetMappingsResponse getMapping(GetMappingsRequest getMappingsRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest, RequestConverters::getMappings, options,
             GetMappingsResponse::fromXContent, emptySet());
     }
@@ -187,8 +187,8 @@ public final class IndicesClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void getMappingsAsync(GetMappingsRequest getMappingsRequest, RequestOptions options,
-                                 ActionListener<GetMappingsResponse> listener) {
+    public void getMappingAsync(GetMappingsRequest getMappingsRequest, RequestOptions options,
+                                ActionListener<GetMappingsResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(getMappingsRequest, RequestConverters::getMappings, options,
             GetMappingsResponse::fromXContent, listener, emptySet());
     }
@@ -474,8 +474,23 @@ public final class IndicesClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
+     * @deprecated use {@link #forcemerge(ForceMergeRequest, RequestOptions)} instead
      */
+    @Deprecated
     public ForceMergeResponse forceMerge(ForceMergeRequest forceMergeRequest, RequestOptions options) throws IOException {
+        return forcemerge(forceMergeRequest, options);
+    }
+
+    /**
+     * Force merge one or more indices using the Force Merge API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html">
+     * Force Merge API on elastic.co</a>
+     * @param forceMergeRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public ForceMergeResponse forcemerge(ForceMergeRequest forceMergeRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(forceMergeRequest, RequestConverters::forceMerge, options,
                 ForceMergeResponse::fromXContent, emptySet());
     }
@@ -487,8 +502,22 @@ public final class IndicesClient {
      * @param forceMergeRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
+     * @deprecated use {@link #forcemergeAsync(ForceMergeRequest, RequestOptions, ActionListener)} instead
      */
+    @Deprecated
     public void forceMergeAsync(ForceMergeRequest forceMergeRequest, RequestOptions options, ActionListener<ForceMergeResponse> listener) {
+        forcemergeAsync(forceMergeRequest, options, listener);
+    }
+
+    /**
+     * Asynchronously force merge one or more indices using the Force Merge API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html">
+     * Force Merge API on elastic.co</a>
+     * @param forceMergeRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void forcemergeAsync(ForceMergeRequest forceMergeRequest, RequestOptions options, ActionListener<ForceMergeResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(forceMergeRequest, RequestConverters::forceMerge, options,
                 ForceMergeResponse::fromXContent, listener, emptySet());
     }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java
index 340e1465397..e889ec5beba 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java
@@ -139,7 +139,7 @@ public final class IngestClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public SimulatePipelineResponse simulatePipeline(SimulatePipelineRequest request, RequestOptions options) throws IOException {
+    public SimulatePipelineResponse simulate(SimulatePipelineRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::simulatePipeline, options,
             SimulatePipelineResponse::fromXContent, emptySet());
     }
@@ -154,9 +154,9 @@ public final class IngestClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void simulatePipelineAsync(SimulatePipelineRequest request,
-                                      RequestOptions options,
-                                      ActionListener<SimulatePipelineResponse> listener) {
+    public void simulateAsync(SimulatePipelineRequest request,
+                              RequestOptions options,
+                              ActionListener<SimulatePipelineResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::simulatePipeline, options,
             SimulatePipelineResponse::fromXContent, listener, emptySet());
     }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index b9e41b87932..c71bebf6903 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -384,8 +384,23 @@ public class RestHighLevelClient implements Closeable {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
+     * @deprecated use {@link #mget(MultiGetRequest, RequestOptions)} instead
      */
+    @Deprecated
     public final MultiGetResponse multiGet(MultiGetRequest multiGetRequest, RequestOptions options) throws IOException {
+        return mget(multiGetRequest, options);
+    }
+
+
+    /**
+     * Retrieves multiple documents by id using the Multi Get API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html">Multi Get API on elastic.co</a>
+     * @param multiGetRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public final MultiGetResponse mget(MultiGetRequest multiGetRequest, RequestOptions options) throws IOException {
         return performRequestAndParseEntity(multiGetRequest, RequestConverters::multiGet, options, MultiGetResponse::fromXContent,
                 singleton(404));
     }
@@ -396,8 +411,21 @@ public class RestHighLevelClient implements Closeable {
      * @param multiGetRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
+     * @deprecated use {@link #mgetAsync(MultiGetRequest, RequestOptions, ActionListener)} instead
      */
+    @Deprecated
     public final void multiGetAsync(MultiGetRequest multiGetRequest, RequestOptions options, ActionListener<MultiGetResponse> listener) {
+        mgetAsync(multiGetRequest, options, listener);
+    }
+
+    /**
+     * Asynchronously retrieves multiple documents by id using the Multi Get API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html">Multi Get API on elastic.co</a>
+     * @param multiGetRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public final void mgetAsync(MultiGetRequest multiGetRequest, RequestOptions options, ActionListener<MultiGetResponse> listener) {
         performRequestAsyncAndParseEntity(multiGetRequest, RequestConverters::multiGet, options, MultiGetResponse::fromXContent, listener,
                 singleton(404));
     }
@@ -531,8 +559,23 @@ public class RestHighLevelClient implements Closeable {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
+     * @deprecated use {@link #msearch(MultiSearchRequest, RequestOptions)} instead
      */
+    @Deprecated
     public final MultiSearchResponse multiSearch(MultiSearchRequest multiSearchRequest, RequestOptions options) throws IOException {
+        return msearch(multiSearchRequest, options);
+    }
+
+    /**
+     * Executes a multi search using the msearch API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html">Multi search API on
+     * elastic.co</a>
+     * @param multiSearchRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public final MultiSearchResponse msearch(MultiSearchRequest multiSearchRequest, RequestOptions options) throws IOException {
         return performRequestAndParseEntity(multiSearchRequest, RequestConverters::multiSearch, options, MultiSearchResponse::fromXContext,
                 emptySet());
     }
@@ -544,9 +587,24 @@ public class RestHighLevelClient implements Closeable {
      * @param searchRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
+     * @deprecated use {@link #msearchAsync(MultiSearchRequest, RequestOptions, ActionListener)} instead
      */
+    @Deprecated
     public final void multiSearchAsync(MultiSearchRequest searchRequest, RequestOptions options,
-                                       ActionListener<MultiSearchResponse> listener) {
+                                   ActionListener<MultiSearchResponse> listener) {
+        msearchAsync(searchRequest, options, listener);
+    }
+
+    /**
+     * Asynchronously executes a multi search using the msearch API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html">Multi search API on
+     * elastic.co</a>
+     * @param searchRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public final void msearchAsync(MultiSearchRequest searchRequest, RequestOptions options,
+                                   ActionListener<MultiSearchResponse> listener) {
         performRequestAsyncAndParseEntity(searchRequest, RequestConverters::multiSearch, options, MultiSearchResponse::fromXContext,
                 listener, emptySet());
     }
@@ -559,8 +617,23 @@ public class RestHighLevelClient implements Closeable {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
+     * @deprecated use {@link #scroll(SearchScrollRequest, RequestOptions)} instead
      */
+    @Deprecated
     public final SearchResponse searchScroll(SearchScrollRequest searchScrollRequest, RequestOptions options) throws IOException {
+        return scroll(searchScrollRequest, options);
+    }
+
+    /**
+     * Executes a search using the Search Scroll API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html">Search Scroll
+     * API on elastic.co</a>
+     * @param searchScrollRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public final SearchResponse scroll(SearchScrollRequest searchScrollRequest, RequestOptions options) throws IOException {
         return performRequestAndParseEntity(searchScrollRequest, RequestConverters::searchScroll, options, SearchResponse::fromXContent,
                 emptySet());
     }
@@ -572,9 +645,24 @@ public class RestHighLevelClient implements Closeable {
      * @param searchScrollRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
+     * @deprecated use {@link #scrollAsync(SearchScrollRequest, RequestOptions, ActionListener)} instead
      */
+    @Deprecated
     public final void searchScrollAsync(SearchScrollRequest searchScrollRequest, RequestOptions options,
-                                        ActionListener<SearchResponse> listener) {
+                                  ActionListener<SearchResponse> listener) {
+        scrollAsync(searchScrollRequest, options, listener);
+    }
+
+    /**
+     * Asynchronously executes a search using the Search Scroll API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html">Search Scroll
+     * API on elastic.co</a>
+     * @param searchScrollRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public final void scrollAsync(SearchScrollRequest searchScrollRequest, RequestOptions options,
+                                  ActionListener<SearchResponse> listener) {
         performRequestAsyncAndParseEntity(searchScrollRequest, RequestConverters::searchScroll, options, SearchResponse::fromXContent,
                 listener, emptySet());
     }
@@ -691,8 +779,8 @@ public class RestHighLevelClient implements Closeable {
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-search-template.html">Multi Search Template API
      * on elastic.co</a>.
      */
-    public final MultiSearchTemplateResponse multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest,
-            RequestOptions options) throws IOException {
+    public final MultiSearchTemplateResponse msearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest,
+                                                             RequestOptions options) throws IOException {
         return performRequestAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
                 options, MultiSearchTemplateResponse::fromXContext, emptySet());
     }
@@ -703,9 +791,9 @@ public class RestHighLevelClient implements Closeable {
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-search-template.html">Multi Search Template API
      * on elastic.co</a>.
      */
-    public final void multiSearchTemplateAsync(MultiSearchTemplateRequest multiSearchTemplateRequest,
-                                          RequestOptions options,
-                                          ActionListener<MultiSearchTemplateResponse> listener) {
+    public final void msearchTemplateAsync(MultiSearchTemplateRequest multiSearchTemplateRequest,
+                                           RequestOptions options,
+                                           ActionListener<MultiSearchTemplateResponse> listener) {
         performRequestAsyncAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
             options, MultiSearchTemplateResponse::fromXContext, listener, emptySet());
     }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
index f75f6cdef24..ae115839bae 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
@@ -63,7 +63,7 @@ public final class SnapshotClient {
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
-    public GetRepositoriesResponse getRepositories(GetRepositoriesRequest getRepositoriesRequest, RequestOptions options)
+    public GetRepositoriesResponse getRepository(GetRepositoriesRequest getRepositoriesRequest, RequestOptions options)
         throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(getRepositoriesRequest, RequestConverters::getRepositories, options,
             GetRepositoriesResponse::fromXContent, emptySet());
@@ -78,8 +78,8 @@ public final class SnapshotClient {
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      */
-    public void getRepositoriesAsync(GetRepositoriesRequest getRepositoriesRequest, RequestOptions options,
-                                     ActionListener<GetRepositoriesResponse> listener) {
+    public void getRepositoryAsync(GetRepositoriesRequest getRepositoriesRequest, RequestOptions options,
+                                   ActionListener<GetRepositoriesResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(getRepositoriesRequest, RequestConverters::getRepositories, options,
             GetRepositoriesResponse::fromXContent, listener, emptySet());
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
index 7605b1c715c..fdd5634ddd6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
@@ -79,7 +79,7 @@ public class BulkProcessorIT extends ESRestHighLevelClientTestCase {
             assertThat(listener.afterCounts.get(), equalTo(1));
             assertThat(listener.bulkFailures.size(), equalTo(0));
             assertResponseItems(listener.bulkItems, numDocs);
-            assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), numDocs);
+            assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
         }
     }
 
@@ -105,7 +105,7 @@ public class BulkProcessorIT extends ESRestHighLevelClientTestCase {
             assertThat(listener.afterCounts.get(), equalTo(1));
             assertThat(listener.bulkFailures.size(), equalTo(0));
             assertResponseItems(listener.bulkItems, numDocs);
-            assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), numDocs);
+            assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
         }
     }
 
@@ -157,7 +157,7 @@ public class BulkProcessorIT extends ESRestHighLevelClientTestCase {
             assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
         }
 
-        assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), numDocs);
+        assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
     }
 
     public void testBulkProcessorWaitOnClose() throws Exception {
@@ -188,7 +188,7 @@ public class BulkProcessorIT extends ESRestHighLevelClientTestCase {
         }
         assertThat(listener.bulkFailures.size(), equalTo(0));
         assertResponseItems(listener.bulkItems, numDocs);
-        assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), numDocs);
+        assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), numDocs);
     }
 
     public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {
@@ -265,7 +265,7 @@ public class BulkProcessorIT extends ESRestHighLevelClientTestCase {
             }
         }
 
-        assertMultiGetResponse(highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT), testDocs);
+        assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), testDocs);
     }
 
     private static MultiGetRequest indexDocs(BulkProcessor processor, int numDocs) throws Exception {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java
index c20998eeb58..5fd9fcb661c 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java
@@ -129,7 +129,7 @@ public class BulkProcessorRetryIT extends ESRestHighLevelClientTestCase {
         }
 
         highLevelClient().indices().refresh(new RefreshRequest(), RequestOptions.DEFAULT);
-        int multiGetResponsesCount = highLevelClient().multiGet(multiGetRequest, RequestOptions.DEFAULT).getResponses().length;
+        int multiGetResponsesCount = highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT).getResponses().length;
 
         if (rejectedExecutionExpected) {
             assertThat(multiGetResponsesCount, lessThanOrEqualTo(numberOfAsyncOps));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
index 9de4c22611c..89f357477fa 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
@@ -253,7 +253,7 @@ public class CrudIT extends ESRestHighLevelClientTestCase {
             MultiGetRequest multiGetRequest = new MultiGetRequest();
             multiGetRequest.add("index", "type", "id1");
             multiGetRequest.add("index", "type", "id2");
-            MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync);
+            MultiGetResponse response = execute(multiGetRequest, highLevelClient()::mget, highLevelClient()::mgetAsync);
             assertEquals(2, response.getResponses().length);
 
             assertTrue(response.getResponses()[0].isFailed());
@@ -285,7 +285,7 @@ public class CrudIT extends ESRestHighLevelClientTestCase {
             MultiGetRequest multiGetRequest = new MultiGetRequest();
             multiGetRequest.add("index", "type", "id1");
             multiGetRequest.add("index", "type", "id2");
-            MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync);
+            MultiGetResponse response = execute(multiGetRequest, highLevelClient()::mget, highLevelClient()::mgetAsync);
             assertEquals(2, response.getResponses().length);
 
             assertFalse(response.getResponses()[0].isFailed());
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java
index 3d1db23da16..ff27fe21c27 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java
@@ -121,7 +121,7 @@ public class CustomRestHighLevelClientTests extends ESTestCase {
      * so that they can be used by subclasses to implement custom logic.
      */
     @SuppressForbidden(reason = "We're forced to uses Class#getDeclaredMethods() here because this test checks protected methods")
-    public void testMethodsVisibility() throws ClassNotFoundException {
+    public void testMethodsVisibility() {
         final String[] methodNames = new String[]{"parseEntity",
                                                   "parseResponseException",
                                                   "performRequest",
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
index 88cf445d436..36a45999b51 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
@@ -443,7 +443,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
             .types("_doc");
 
         GetMappingsResponse getMappingsResponse =
-            execute(request, highLevelClient().indices()::getMappings, highLevelClient().indices()::getMappingsAsync);
+            execute(request, highLevelClient().indices()::getMapping, highLevelClient().indices()::getMappingAsync);
 
         Map<String, Object> mappings = getMappingsResponse.getMappings().get(indexName).get("_doc").sourceAsMap();
         Map<String, String> type = new HashMap<>();
@@ -796,7 +796,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
             createIndex(index, settings);
             ForceMergeRequest forceMergeRequest = new ForceMergeRequest(index);
             ForceMergeResponse forceMergeResponse =
-                execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync);
+                execute(forceMergeRequest, highLevelClient().indices()::forcemerge, highLevelClient().indices()::forcemergeAsync);
             assertThat(forceMergeResponse.getTotalShards(), equalTo(1));
             assertThat(forceMergeResponse.getSuccessfulShards(), equalTo(1));
             assertThat(forceMergeResponse.getFailedShards(), equalTo(0));
@@ -807,7 +807,7 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
             assertFalse(indexExists(nonExistentIndex));
             ForceMergeRequest forceMergeRequest = new ForceMergeRequest(nonExistentIndex);
             ElasticsearchException exception = expectThrows(ElasticsearchException.class,
-                () -> execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync));
+                () -> execute(forceMergeRequest, highLevelClient().indices()::forcemerge, highLevelClient().indices()::forcemergeAsync));
             assertEquals(RestStatus.NOT_FOUND, exception.status());
         }
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java
index 6fd6f950595..1f5914f392c 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java
@@ -135,7 +135,7 @@ public class IngestClientIT extends ESRestHighLevelClientTestCase {
         );
         request.setVerbose(isVerbose);
         SimulatePipelineResponse response =
-            execute(request, highLevelClient().ingest()::simulatePipeline, highLevelClient().ingest()::simulatePipelineAsync);
+            execute(request, highLevelClient().ingest()::simulate, highLevelClient().ingest()::simulateAsync);
         List<SimulateDocumentResult> results = response.getResults();
         assertEquals(1, results.size());
         if (isVerbose) {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 2925062e0e7..47870125aa2 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
-import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
@@ -53,6 +51,7 @@ import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -73,20 +72,30 @@ import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregation
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalAggregationTestCase;
+import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
+import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
 import java.net.SocketTimeoutException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
-import static org.elasticsearch.client.RestClientTestUtil.randomHeaders;
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.hamcrest.CoreMatchers.endsWith;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
@@ -137,7 +146,6 @@ public class RestHighLevelClientTests extends ESTestCase {
     }
 
     public void testInfo() throws IOException {
-        Header[] headers = randomHeaders(random(), "Header");
         MainResponse testInfo = new MainResponse("nodeName", Version.CURRENT, new ClusterName("clusterName"), "clusterUuid",
                 Build.CURRENT);
         mockResponse(testInfo);
@@ -150,7 +158,7 @@ public class RestHighLevelClientTests extends ESTestCase {
                 null, false, false, null, 1), randomAlphaOfLengthBetween(5, 10), 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
                 SearchResponse.Clusters.EMPTY);
         mockResponse(mockSearchResponse);
-        SearchResponse searchResponse = restHighLevelClient.searchScroll(
+        SearchResponse searchResponse = restHighLevelClient.scroll(
                 new SearchScrollRequest(randomAlphaOfLengthBetween(5, 10)), RequestOptions.DEFAULT);
         assertEquals(mockSearchResponse.getScrollId(), searchResponse.getScrollId());
         assertEquals(0, searchResponse.getHits().totalHits);
@@ -632,6 +640,149 @@ public class RestHighLevelClientTests extends ESTestCase {
         assertTrue(names.contains(DiscountedCumulativeGain.NAME));
     }
 
+    public void testApiNamingConventions() throws Exception {
+        //this list should be empty once the high-level client is feature complete
+        String[] notYetSupportedApi = new String[]{
+            "cluster.remote_info",
+            "count",
+            "create",
+            "delete_by_query",
+            "exists_source",
+            "get_source",
+            "indices.delete_alias",
+            "indices.delete_template",
+            "indices.exists_template",
+            "indices.exists_type",
+            "indices.get_upgrade",
+            "indices.put_alias",
+            "mtermvectors",
+            "put_script",
+            "reindex",
+            "reindex_rethrottle",
+            "render_search_template",
+            "scripts_painless_execute",
+            "snapshot.restore",
+            "tasks.get",
+            "termvectors",
+            "update_by_query"
+        };
+        //These API are not required for high-level client feature completeness
+        String[] notRequiredApi = new String[] {
+            "cluster.allocation_explain",
+            "cluster.pending_tasks",
+            "cluster.reroute",
+            "cluster.state",
+            "cluster.stats",
+            "indices.shard_stores",
+            "indices.upgrade",
+            "indices.recovery",
+            "indices.segments",
+            "indices.stats",
+            "ingest.processor_grok",
+            "nodes.info",
+            "nodes.stats",
+            "nodes.hot_threads",
+            "nodes.usage",
+            "search_shards",
+        };
+        Set<String> deprecatedMethods = new HashSet<>();
+        deprecatedMethods.add("indices.force_merge");
+        deprecatedMethods.add("multi_get");
+        deprecatedMethods.add("multi_search");
+        deprecatedMethods.add("search_scroll");
+
+        ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load("/rest-api-spec/api");
+        Set<String> apiSpec = restSpec.getApis().stream().map(ClientYamlSuiteRestApi::getName).collect(Collectors.toSet());
+
+        Set<String> topLevelMethodsExclusions = new HashSet<>();
+        topLevelMethodsExclusions.add("getLowLevelClient");
+        topLevelMethodsExclusions.add("close");
+
+        Map<String, Method> methods = Arrays.stream(RestHighLevelClient.class.getMethods())
+                .filter(method -> method.getDeclaringClass().equals(RestHighLevelClient.class)
+                        && topLevelMethodsExclusions.contains(method.getName()) == false)
+                .map(method -> Tuple.tuple(toSnakeCase(method.getName()), method))
+                .flatMap(tuple -> tuple.v2().getReturnType().getName().endsWith("Client")
+                        ? getSubClientMethods(tuple.v1(), tuple.v2().getReturnType()) : Stream.of(tuple))
+                .collect(Collectors.toMap(Tuple::v1, Tuple::v2));
+
+        Set<String> apiNotFound = new HashSet<>();
+
+        for (Map.Entry<String, Method> entry : methods.entrySet()) {
+            Method method = entry.getValue();
+            String apiName = entry.getKey();
+
+            assertTrue("method [" + apiName + "] is not final",
+                    Modifier.isFinal(method.getClass().getModifiers()) || Modifier.isFinal(method.getModifiers()));
+            assertTrue(Modifier.isPublic(method.getModifiers()));
+
+            //we convert all the method names to snake case, hence we need to look for the '_async' suffix rather than 'Async'
+            if (apiName.endsWith("_async")) {
+                assertTrue("async method [" + method.getName() + "] doesn't have corresponding sync method",
+                        methods.containsKey(apiName.substring(0, apiName.length() - 6)));
+                assertThat(method.getReturnType(), equalTo(Void.TYPE));
+                assertEquals(0, method.getExceptionTypes().length);
+                assertEquals(3, method.getParameterTypes().length);
+                assertThat(method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
+                assertThat(method.getParameterTypes()[1].getName(), equalTo(RequestOptions.class.getName()));
+                assertThat(method.getParameterTypes()[2].getName(), equalTo(ActionListener.class.getName()));
+            } else {
+                //A few methods return a boolean rather than a response object
+                if (apiName.equals("ping") || apiName.contains("exist")) {
+                    assertThat(method.getReturnType().getSimpleName(), equalTo("boolean"));
+                } else {
+                    assertThat(method.getReturnType().getSimpleName(), endsWith("Response"));
+                }
+
+                assertEquals(1, method.getExceptionTypes().length);
+                //a few methods don't accept a request object as argument
+                if (apiName.equals("ping") || apiName.equals("info")) {
+                    assertEquals(1, method.getParameterTypes().length);
+                    assertThat(method.getParameterTypes()[0].getName(), equalTo(RequestOptions.class.getName()));
+                } else {
+                    assertEquals(apiName, 2, method.getParameterTypes().length);
+                    assertThat(method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
+                    assertThat(method.getParameterTypes()[1].getName(), equalTo(RequestOptions.class.getName()));
+                }
+
+                boolean remove = apiSpec.remove(apiName);
+                if (remove == false && deprecatedMethods.contains(apiName) == false) {
+                    //TODO xpack api are currently ignored, we need to load xpack yaml spec too
+                    if (apiName.startsWith("xpack.") == false) {
+                        apiNotFound.add(apiName);
+                    }
+                }
+            }
+        }
+        assertThat("Some client method doesn't match a corresponding API defined in the REST spec: " + apiNotFound,
+            apiNotFound.size(), equalTo(0));
+
+        //we decided not to support cat API in the high-level REST client, they are supposed to be used from a low-level client
+        apiSpec.removeIf(api -> api.startsWith("cat."));
+        Stream.concat(Arrays.stream(notYetSupportedApi), Arrays.stream(notRequiredApi)).forEach(
+            api -> assertTrue(api + " API is either not defined in the spec or already supported by the high-level client",
+                apiSpec.remove(api)));
+        assertThat("Some API are not supported but they should be: " + apiSpec, apiSpec.size(), equalTo(0));
+    }
+
+    private static Stream<Tuple<String, Method>> getSubClientMethods(String namespace, Class<?> clientClass) {
+        return Arrays.stream(clientClass.getMethods()).filter(method -> method.getDeclaringClass().equals(clientClass))
+                .map(method -> Tuple.tuple(namespace + "." + toSnakeCase(method.getName()), method));
+    }
+
+    private static String toSnakeCase(String camelCase) {
+        StringBuilder snakeCaseString = new StringBuilder();
+        for (Character aChar : camelCase.toCharArray()) {
+            if (Character.isUpperCase(aChar)) {
+                snakeCaseString.append('_');
+                snakeCaseString.append(Character.toLowerCase(aChar));
+            } else {
+                snakeCaseString.append(aChar);
+            }
+        }
+        return snakeCaseString.toString();
+    }
+
     private static class TrackingActionListener implements ActionListener<Integer> {
         private final AtomicInteger statusCode = new AtomicInteger(-1);
         private final AtomicReference<Exception> exception = new AtomicReference<>();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
index ce9091a91ff..9c9c5425f00 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
@@ -597,7 +597,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
             }
 
             searchResponse = execute(new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)),
-                    highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync);
+                    highLevelClient()::scroll, highLevelClient()::scrollAsync);
 
             assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(35));
@@ -606,7 +606,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
             }
 
             searchResponse = execute(new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)),
-                    highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync);
+                    highLevelClient()::scroll, highLevelClient()::scrollAsync);
 
             assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L));
             assertThat(searchResponse.getHits().getHits().length, equalTo(30));
@@ -623,7 +623,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
 
             SearchScrollRequest scrollRequest = new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2));
             ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> execute(scrollRequest,
-                    highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync));
+                    highLevelClient()::scroll, highLevelClient()::scrollAsync));
             assertEquals(RestStatus.NOT_FOUND, exception.status());
             assertThat(exception.getRootCause(), instanceOf(ElasticsearchException.class));
             ElasticsearchException rootCause = (ElasticsearchException) exception.getRootCause();
@@ -644,7 +644,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
         multiSearchRequest.add(searchRequest3);
 
         MultiSearchResponse multiSearchResponse =
-                execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+                execute(multiSearchRequest, highLevelClient()::msearch, highLevelClient()::msearchAsync);
         assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
         assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3));
 
@@ -686,7 +686,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
         multiSearchRequest.add(searchRequest3);
 
         MultiSearchResponse multiSearchResponse =
-                execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+                execute(multiSearchRequest, highLevelClient()::msearch, highLevelClient()::msearchAsync);
         assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
         assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3));
 
@@ -734,7 +734,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
         multiSearchRequest.add(searchRequest3);
 
         MultiSearchResponse multiSearchResponse =
-                execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+                execute(multiSearchRequest, highLevelClient()::msearch, highLevelClient()::msearchAsync);
         assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
         assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3));
 
@@ -759,7 +759,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
         searchRequest1.source().highlighter(new HighlightBuilder().field("field"));
         searchRequest2.source().highlighter(new HighlightBuilder().field("field"));
         searchRequest3.source().highlighter(new HighlightBuilder().field("field"));
-        multiSearchResponse = execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+        multiSearchResponse = execute(multiSearchRequest, highLevelClient()::msearch, highLevelClient()::msearchAsync);
         assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
         assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3));
 
@@ -797,7 +797,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
         multiSearchRequest.add(searchRequest2);
 
         MultiSearchResponse multiSearchResponse =
-                execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+                execute(multiSearchRequest, highLevelClient()::msearch, highLevelClient()::msearchAsync);
         assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
         assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(2));
 
@@ -941,8 +941,8 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
         multiSearchTemplateRequest.add(badRequest);
 
         MultiSearchTemplateResponse multiSearchTemplateResponse =
-                execute(multiSearchTemplateRequest, highLevelClient()::multiSearchTemplate,
-                        highLevelClient()::multiSearchTemplateAsync);
+                execute(multiSearchTemplateRequest, highLevelClient()::msearchTemplate,
+                        highLevelClient()::msearchTemplateAsync);
 
         Item[] responses = multiSearchTemplateResponse.getResponses();
 
@@ -999,8 +999,8 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
 
         // The whole HTTP request should fail if no nested search requests are valid
         ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class,
-                () -> execute(multiSearchTemplateRequest, highLevelClient()::multiSearchTemplate,
-                        highLevelClient()::multiSearchTemplateAsync));
+                () -> execute(multiSearchTemplateRequest, highLevelClient()::msearchTemplate,
+                        highLevelClient()::msearchTemplateAsync));
 
         assertEquals(RestStatus.BAD_REQUEST, exception.status());
         assertThat(exception.getMessage(), containsString("no requests added"));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
index 6d035f5db65..5483f055c2c 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
@@ -77,8 +77,8 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
 
         GetRepositoriesRequest request = new GetRepositoriesRequest();
         request.repositories(new String[]{testRepository});
-        GetRepositoriesResponse response = execute(request, highLevelClient().snapshot()::getRepositories,
-            highLevelClient().snapshot()::getRepositoriesAsync);
+        GetRepositoriesResponse response = execute(request, highLevelClient().snapshot()::getRepository,
+            highLevelClient().snapshot()::getRepositoryAsync);
         assertThat(1, equalTo(response.repositories().size()));
     }
 
@@ -86,8 +86,8 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         assertTrue(createTestRepository("other", FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
         assertTrue(createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
 
-        GetRepositoriesResponse response = execute(new GetRepositoriesRequest(), highLevelClient().snapshot()::getRepositories,
-            highLevelClient().snapshot()::getRepositoriesAsync);
+        GetRepositoriesResponse response = execute(new GetRepositoriesRequest(), highLevelClient().snapshot()::getRepository,
+            highLevelClient().snapshot()::getRepositoryAsync);
         assertThat(2, equalTo(response.repositories().size()));
     }
 
@@ -95,7 +95,7 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         String repository = "doesnotexist";
         GetRepositoriesRequest request = new GetRepositoriesRequest(new String[]{repository});
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(request,
-            highLevelClient().snapshot()::getRepositories, highLevelClient().snapshot()::getRepositoriesAsync));
+            highLevelClient().snapshot()::getRepository, highLevelClient().snapshot()::getRepositoryAsync));
 
         assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
         assertThat(exception.getMessage(), equalTo(
@@ -107,8 +107,8 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         assertTrue(createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
 
         GetRepositoriesRequest request = new GetRepositoriesRequest();
-        GetRepositoriesResponse response = execute(request, highLevelClient().snapshot()::getRepositories,
-            highLevelClient().snapshot()::getRepositoriesAsync);
+        GetRepositoriesResponse response = execute(request, highLevelClient().snapshot()::getRepository,
+            highLevelClient().snapshot()::getRepositoryAsync);
         assertThat(1, equalTo(response.repositories().size()));
 
         DeleteRepositoryRequest deleteRequest = new DeleteRepositoryRequest(repository);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
index 9dad115643c..ad41c139ddc 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
@@ -1121,7 +1121,7 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase {
             // end::multi-get-request-top-level-extras
 
             // tag::multi-get-execute
-            MultiGetResponse response = client.multiGet(request, RequestOptions.DEFAULT);
+            MultiGetResponse response = client.mget(request, RequestOptions.DEFAULT);
             // end::multi-get-execute
 
             // tag::multi-get-response
@@ -1174,7 +1174,7 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::multi-get-execute-async
-            client.multiGetAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.mgetAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::multi-get-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
@@ -1185,7 +1185,7 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase {
             request.add(new MultiGetRequest.Item("index", "type", "example_id")
                 .fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE));  // <1>
             // end::multi-get-request-no-source
-            MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request, RequestOptions.DEFAULT));
+            MultiGetItemResponse item = unwrapAndAssertExample(client.mget(request, RequestOptions.DEFAULT));
             assertNull(item.getResponse().getSource());
         }
         {
@@ -1198,7 +1198,7 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase {
             request.add(new MultiGetRequest.Item("index", "type", "example_id")
                 .fetchSourceContext(fetchSourceContext));  // <1>
             // end::multi-get-request-source-include
-            MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request, RequestOptions.DEFAULT));
+            MultiGetItemResponse item = unwrapAndAssertExample(client.mget(request, RequestOptions.DEFAULT));
             assertThat(item.getResponse().getSource(), hasEntry("foo", "val1"));
             assertThat(item.getResponse().getSource(), hasEntry("bar", "val2"));
             assertThat(item.getResponse().getSource(), not(hasKey("baz")));
@@ -1213,7 +1213,7 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase {
             request.add(new MultiGetRequest.Item("index", "type", "example_id")
                 .fetchSourceContext(fetchSourceContext));  // <1>
             // end::multi-get-request-source-exclude
-            MultiGetItemResponse item = unwrapAndAssertExample(client.multiGet(request, RequestOptions.DEFAULT));
+            MultiGetItemResponse item = unwrapAndAssertExample(client.mget(request, RequestOptions.DEFAULT));
             assertThat(item.getResponse().getSource(), not(hasKey("foo")));
             assertThat(item.getResponse().getSource(), not(hasKey("bar")));
             assertThat(item.getResponse().getSource(), hasEntry("baz", "val3"));
@@ -1223,7 +1223,7 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase {
             // tag::multi-get-request-stored
             request.add(new MultiGetRequest.Item("index", "type", "example_id")
                 .storedFields("foo"));  // <1>
-            MultiGetResponse response = client.multiGet(request, RequestOptions.DEFAULT);
+            MultiGetResponse response = client.mget(request, RequestOptions.DEFAULT);
             MultiGetItemResponse item = response.getResponses()[0];
             String value = item.getResponse().getField("foo").getValue(); // <2>
             // end::multi-get-request-stored
@@ -1235,7 +1235,7 @@ public class CRUDDocumentationIT extends ESRestHighLevelClientTestCase {
             MultiGetRequest request = new MultiGetRequest();
             request.add(new MultiGetRequest.Item("index", "type", "example_id")
                 .version(1000L));
-            MultiGetResponse response = client.multiGet(request, RequestOptions.DEFAULT);
+            MultiGetResponse response = client.mget(request, RequestOptions.DEFAULT);
             MultiGetItemResponse item = response.getResponses()[0];
             assertNull(item.getResponse());                          // <1>
             Exception e = item.getFailure().getFailure();            // <2>
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
index 23dab5b21e2..36d562c501f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
@@ -622,7 +622,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::get-mapping-request-indicesOptions
 
             // tag::get-mapping-execute
-            GetMappingsResponse getMappingResponse = client.indices().getMappings(request, RequestOptions.DEFAULT);
+            GetMappingsResponse getMappingResponse = client.indices().getMapping(request, RequestOptions.DEFAULT);
             // end::get-mapping-execute
 
             // tag::get-mapping-response
@@ -704,7 +704,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             });
 
             // tag::get-mapping-execute-async
-            client.indices().getMappingsAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.indices().getMappingAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::get-mapping-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
@@ -1344,7 +1344,7 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::force-merge-request-flush
 
             // tag::force-merge-execute
-            ForceMergeResponse forceMergeResponse = client.indices().forceMerge(request, RequestOptions.DEFAULT);
+            ForceMergeResponse forceMergeResponse = client.indices().forcemerge(request, RequestOptions.DEFAULT);
             // end::force-merge-execute
 
             // tag::force-merge-response
@@ -1369,14 +1369,14 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::force-merge-execute-listener
 
             // tag::force-merge-execute-async
-            client.indices().forceMergeAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.indices().forcemergeAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::force-merge-execute-async
         }
         {
             // tag::force-merge-notfound
             try {
                 ForceMergeRequest request = new ForceMergeRequest("does_not_exist");
-                client.indices().forceMerge(request, RequestOptions.DEFAULT);
+                client.indices().forcemerge(request, RequestOptions.DEFAULT);
             } catch (ElasticsearchException exception) {
                 if (exception.status() == RestStatus.NOT_FOUND) {
                     // <1>
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java
index c53ec2b5d7c..98502e3668a 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java
@@ -317,7 +317,7 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
             // end::simulate-pipeline-request-verbose
 
             // tag::simulate-pipeline-execute
-            SimulatePipelineResponse response = client.ingest().simulatePipeline(request, RequestOptions.DEFAULT); // <1>
+            SimulatePipelineResponse response = client.ingest().simulate(request, RequestOptions.DEFAULT); // <1>
             // end::simulate-pipeline-execute
 
             // tag::simulate-pipeline-response
@@ -381,7 +381,7 @@ public class IngestClientDocumentationIT extends ESRestHighLevelClientTestCase {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::simulate-pipeline-execute-async
-            client.ingest().simulatePipelineAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.ingest().simulateAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::simulate-pipeline-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
index 26bb4682fd9..c60f2d4c92b 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
@@ -583,7 +583,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
             // tag::search-scroll2
             SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // <1>
             scrollRequest.scroll(TimeValue.timeValueSeconds(30));
-            SearchResponse searchScrollResponse = client.searchScroll(scrollRequest, RequestOptions.DEFAULT);
+            SearchResponse searchScrollResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
             scrollId = searchScrollResponse.getScrollId();  // <2>
             hits = searchScrollResponse.getHits(); // <3>
             assertEquals(3, hits.getTotalHits());
@@ -612,7 +612,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
             // end::scroll-request-arguments
 
             // tag::search-scroll-execute-sync
-            SearchResponse searchResponse = client.searchScroll(scrollRequest, RequestOptions.DEFAULT);
+            SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
             // end::search-scroll-execute-sync
 
             assertEquals(0, searchResponse.getFailedShards());
@@ -638,7 +638,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
             scrollListener = new LatchedActionListener<>(scrollListener, latch);
 
             // tag::search-scroll-execute-async
-            client.searchScrollAsync(scrollRequest, RequestOptions.DEFAULT, scrollListener); // <1>
+            client.scrollAsync(scrollRequest, RequestOptions.DEFAULT, scrollListener); // <1>
             // end::search-scroll-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
@@ -710,7 +710,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
             while (searchHits != null && searchHits.length > 0) { // <2>
                 SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // <3>
                 scrollRequest.scroll(scroll);
-                searchResponse = client.searchScroll(scrollRequest, RequestOptions.DEFAULT);
+                searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
                 scrollId = searchResponse.getScrollId();
                 searchHits = searchResponse.getHits().getHits();
                 // <4>
@@ -861,7 +861,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
         // end::multi-search-template-request-inline
 
         // tag::multi-search-template-request-sync
-        MultiSearchTemplateResponse multiResponse = client.multiSearchTemplate(multiRequest, RequestOptions.DEFAULT);
+        MultiSearchTemplateResponse multiResponse = client.msearchTemplate(multiRequest, RequestOptions.DEFAULT);
         // end::multi-search-template-request-sync
 
         // tag::multi-search-template-response
@@ -916,7 +916,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
 
 
         // tag::multi-search-template-execute
-        MultiSearchTemplateResponse multiResponse = client.multiSearchTemplate(multiRequest, RequestOptions.DEFAULT);
+        MultiSearchTemplateResponse multiResponse = client.msearchTemplate(multiRequest, RequestOptions.DEFAULT);
         // end::multi-search-template-execute
 
         assertNotNull(multiResponse);
@@ -944,7 +944,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
         listener = new LatchedActionListener<>(listener, latch);
 
         // tag::multi-search-template-execute-async
-        client.multiSearchTemplateAsync(multiRequest, RequestOptions.DEFAULT, listener);
+        client.msearchTemplateAsync(multiRequest, RequestOptions.DEFAULT, listener);
         // end::multi-search-template-execute-async
 
         assertTrue(latch.await(30L, TimeUnit.SECONDS));
@@ -1201,7 +1201,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
             request.add(secondSearchRequest);
             // end::multi-search-request-basic
             // tag::multi-search-execute
-            MultiSearchResponse response = client.multiSearch(request, RequestOptions.DEFAULT);
+            MultiSearchResponse response = client.msearch(request, RequestOptions.DEFAULT);
             // end::multi-search-execute
             // tag::multi-search-response
             MultiSearchResponse.Item firstResponse = response.getResponses()[0];   // <1>
@@ -1233,7 +1233,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::multi-search-execute-async
-            client.multiSearchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.msearchAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::multi-search-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
@@ -1244,7 +1244,7 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
             request.add(new SearchRequest("posts")  // <1>
                     .types("doc"));                 // <2>
             // end::multi-search-request-index
-            MultiSearchResponse response = client.multiSearch(request, RequestOptions.DEFAULT);
+            MultiSearchResponse response = client.msearch(request, RequestOptions.DEFAULT);
             MultiSearchResponse.Item firstResponse = response.getResponses()[0];
             assertNull(firstResponse.getFailure());
             SearchResponse searchResponse = firstResponse.getResponse();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
index 68a8113af6d..fff3e7ece70 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
@@ -221,7 +221,7 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
         // end::get-repository-request-masterTimeout
 
         // tag::get-repository-execute
-        GetRepositoriesResponse response = client.snapshot().getRepositories(request, RequestOptions.DEFAULT);
+        GetRepositoriesResponse response = client.snapshot().getRepository(request, RequestOptions.DEFAULT);
         // end::get-repository-execute
 
         // tag::get-repository-response
@@ -256,7 +256,7 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::get-repository-execute-async
-            client.snapshot().getRepositoriesAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.snapshot().getRepositoryAsync(request, RequestOptions.DEFAULT, listener); // <1>
             // end::get-repository-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
diff --git a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
index 29aec900cef..6bfa4de8d4a 100644
--- a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
+++ b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java
@@ -171,7 +171,7 @@ public class CrossClusterSearchUnavailableClusterIT extends ESRestTestCase {
                 assertEquals(10, response.getHits().totalHits);
                 assertEquals(10, response.getHits().getHits().length);
                 String scrollId = response.getScrollId();
-                SearchResponse scrollResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(scrollId), RequestOptions.DEFAULT);
+                SearchResponse scrollResponse = restHighLevelClient.scroll(new SearchScrollRequest(scrollId), RequestOptions.DEFAULT);
                 assertSame(SearchResponse.Clusters.EMPTY, scrollResponse.getClusters());
                 assertEquals(10, scrollResponse.getHits().totalHits);
                 assertEquals(0, scrollResponse.getHits().getHits().length);
@@ -206,7 +206,7 @@ public class CrossClusterSearchUnavailableClusterIT extends ESRestTestCase {
                 assertEquals(10, response.getHits().totalHits);
                 assertEquals(10, response.getHits().getHits().length);
                 String scrollId = response.getScrollId();
-                SearchResponse scrollResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(scrollId), RequestOptions.DEFAULT);
+                SearchResponse scrollResponse = restHighLevelClient.scroll(new SearchScrollRequest(scrollId), RequestOptions.DEFAULT);
                 assertSame(SearchResponse.Clusters.EMPTY, scrollResponse.getClusters());
                 assertEquals(10, scrollResponse.getHits().totalHits);
                 assertEquals(0, scrollResponse.getHits().getHits().length);

From b655c11dbee65d5331819f04a15ace80b639cfbd Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Tue, 17 Jul 2018 10:34:45 +0100
Subject: [PATCH 057/260] =?UTF-8?q?Mute=20:qa:mixed-cluster=20indices.stat?=
 =?UTF-8?q?s/10=5Findex/Index=20-=20all=E2=80=99?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 qa/mixed-cluster/build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
index ac57d51def7..db743cb12b1 100644
--- a/qa/mixed-cluster/build.gradle
+++ b/qa/mixed-cluster/build.gradle
@@ -57,6 +57,7 @@ for (Version version : bwcVersions.wireCompatible) {
   tasks.getByName("${baseName}#mixedClusterTestRunner").configure {
     /* To support taking index snapshots, we have to set path.repo setting */
     systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
+    systemProperty 'tests.rest.blacklist', ['indices.stats/10_index/Index - all'].join(',')
   }
 }
 

From b43fe560a43ee5c43bae5105b07aae811a69b1ef Mon Sep 17 00:00:00 2001
From: Alpar Torok <torokalpar@gmail.com>
Date: Tue, 17 Jul 2018 11:41:31 +0000
Subject: [PATCH 058/260] Updates the build to gradle 4.9 (#32087)

There are fixes to the dependency report, most importantly for us,
it still works even if `failOnVersionConflict` would fail the build.
---
 build.gradle                                                  | 2 +-
 .../carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy | 2 +-
 gradle/wrapper/gradle-wrapper.properties                      | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build.gradle b/build.gradle
index a75d093664f..187e2477052 100644
--- a/build.gradle
+++ b/build.gradle
@@ -596,7 +596,7 @@ if (System.properties.get("build.compare") != null) {
       }
     }
     sourceBuild {
-      gradleVersion = "4.8.1" // does not default to gradle weapper of project dir, but current version
+      gradleVersion = gradle.getGradleVersion()
       projectDir = referenceProject
       tasks = ["clean", "assemble"]
       arguments = ["-Dbuild.compare_friendly=true"]
diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
index 24b66efbcef..d4c8f89bf50 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
@@ -74,7 +74,7 @@ class RandomizedTestingPlugin implements Plugin<Project> {
         // since we can't be sure if the task was ever realized, we remove both the provider and the task
         TaskProvider<Test> oldTestProvider
         try {
-            oldTestProvider = tasks.getByNameLater(Test, 'test')
+            oldTestProvider = tasks.named('test')
         } catch (UnknownTaskException unused) {
             // no test task, ok, user will use testing task on their own
             return
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 37e3d3699fa..94161917d18 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=ce1645ff129d11aad62dab70d63426fdce6cfd646fa309dc5dc5255dd03c7c11
+distributionSha256Sum=39e2d5803bbd5eaf6c8efe07067b0e5a00235e8c71318642b2ed262920b27721

From a7e477126f5b9c18decf24314cacd2ad5456b2d5 Mon Sep 17 00:00:00 2001
From: markharwood <markharwood@gmail.com>
Date: Tue, 17 Jul 2018 13:11:10 +0100
Subject: [PATCH 059/260] Relax TermVectors API to work with textual fields
 other than TextFieldType (#31915)

This changes the field-eligibility test to check one level up in the class hierarchy to allow any subclasses of StringFieldType.
Closes #31902
---
 .../index/termvectors/TermVectorsService.java |  5 +-
 .../termvectors/TermVectorsServiceTests.java  | 54 ++++++++++++++++++-
 2 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index c13c56beb5a..bc77626b942 100644
--- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -48,7 +48,7 @@ import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
@@ -162,8 +162,7 @@ public class TermVectorsService  {
 
     private static boolean isValidField(MappedFieldType fieldType) {
         // must be a string
-        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType == false
-                && fieldType instanceof TextFieldMapper.TextFieldType == false) {
+        if (fieldType instanceof StringFieldType == false) {
             return false;
         }
         // and must be indexed
diff --git a/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java b/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
index e5adbde71eb..034e7daaf7f 100644
--- a/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
@@ -109,12 +109,62 @@ public class TermVectorsServiceTests extends ESSingleNodeTestCase {
         IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         assertThat(shard, notNullValue());
-        TermVectorsResponse response = TermVectorsService.getTermVectors(shard, request);
+        TermVectorsResponse response = TermVectorsService.getTermVectors(shard, request);        
+        assertEquals(1, response.getFields().size());            
 
         Terms terms = response.getFields().terms("text");
         TermsEnum iterator = terms.iterator();
         while (iterator.next() != null) {
             assertEquals(max, iterator.docFreq());
         }
-    }
+    }  
+    
+    public void testWithIndexedPhrases() throws IOException {
+        XContentBuilder mapping = jsonBuilder()
+            .startObject()
+                .startObject("_doc")
+                    .startObject("properties")
+                        .startObject("text")
+                            .field("type", "text")
+                            .field("index_phrases", true)                            
+                            .field("term_vector", "with_positions_offsets_payloads")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        Settings settings = Settings.builder()
+                .put("number_of_shards", 1)
+                .build();
+        createIndex("test", settings, "_doc", mapping);
+        ensureGreen();
+
+        int max = between(3, 10);
+        BulkRequestBuilder bulk = client().prepareBulk();
+        for (int i = 0; i < max; i++) {
+            bulk.add(client().prepareIndex("test", "_doc", Integer.toString(i))
+                    .setSource("text", "the quick brown fox jumped over the lazy dog"));
+        }
+        bulk.get();
+
+        TermVectorsRequest request = new TermVectorsRequest("test", "_doc", "0").termStatistics(true);
+
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService test = indicesService.indexService(resolveIndex("test"));
+        IndexShard shard = test.getShardOrNull(0);
+        assertThat(shard, notNullValue());
+        TermVectorsResponse response = TermVectorsService.getTermVectors(shard, request);        
+        assertEquals(2, response.getFields().size());
+
+        Terms terms = response.getFields().terms("text");
+        TermsEnum iterator = terms.iterator();
+        while (iterator.next() != null) {
+            assertEquals(max, iterator.docFreq());
+        }
+                
+        Terms phrases = response.getFields().terms("text._index_phrase");
+        TermsEnum phraseIterator = phrases.iterator();
+        while (phraseIterator.next() != null) {
+            assertEquals(max, phraseIterator.docFreq());
+        }
+    }  
 }

From ed3b44fb4cd6f957facf6d0f38daa08eeecd52ea Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Tue, 17 Jul 2018 14:14:02 +0200
Subject: [PATCH 060/260] Handle TokenizerFactory  TODOs (#32063)

* Don't replace Replace TokenizerFactory with Supplier, this approach was rejected in #32063
* Remove unused parameter from constructor
---
 .../analysis/common/CharGroupTokenizerFactory.java             | 2 +-
 .../elasticsearch/analysis/common/ClassicTokenizerFactory.java | 2 +-
 .../analysis/common/EdgeNGramTokenizerFactory.java             | 2 +-
 .../elasticsearch/analysis/common/KeywordTokenizerFactory.java | 2 +-
 .../elasticsearch/analysis/common/LetterTokenizerFactory.java  | 2 +-
 .../analysis/common/LowerCaseTokenizerFactory.java             | 2 +-
 .../elasticsearch/analysis/common/NGramTokenizerFactory.java   | 2 +-
 .../analysis/common/PathHierarchyTokenizerFactory.java         | 2 +-
 .../elasticsearch/analysis/common/PatternTokenizerFactory.java | 2 +-
 .../analysis/common/SimplePatternSplitTokenizerFactory.java    | 2 +-
 .../analysis/common/SimplePatternTokenizerFactory.java         | 2 +-
 .../elasticsearch/analysis/common/ThaiTokenizerFactory.java    | 2 +-
 .../analysis/common/UAX29URLEmailTokenizerFactory.java         | 2 +-
 .../analysis/common/WhitespaceTokenizerFactory.java            | 2 +-
 .../org/elasticsearch/index/analysis/IcuTokenizerFactory.java  | 2 +-
 .../elasticsearch/index/analysis/KuromojiTokenizerFactory.java | 2 +-
 .../org/elasticsearch/index/analysis/NoriTokenizerFactory.java | 2 +-
 .../index/analysis/SmartChineseTokenizerTokenizerFactory.java  | 2 +-
 .../elasticsearch/index/analysis/AbstractTokenizerFactory.java | 3 +--
 .../elasticsearch/index/analysis/StandardTokenizerFactory.java | 2 +-
 .../org/elasticsearch/index/analysis/TokenizerFactory.java     | 2 +-
 .../xpack/ml/job/categorization/MlClassicTokenizerFactory.java | 2 +-
 22 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactory.java
index d4e1e794a30..4920b7daae8 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CharGroupTokenizerFactory.java
@@ -39,7 +39,7 @@ public class CharGroupTokenizerFactory extends AbstractTokenizerFactory{
     private boolean tokenizeOnSymbol = false;
 
     public CharGroupTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
 
         for (final String c : settings.getAsList("tokenize_on_chars")) {
             if (c == null || c.length() == 0) {
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java
index e81f6b88d24..27316f4cde5 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java
@@ -35,7 +35,7 @@ public class ClassicTokenizerFactory extends AbstractTokenizerFactory {
     private final int maxTokenLength;
 
     ClassicTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         maxTokenLength = settings.getAsInt("max_token_length", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
     }
 
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerFactory.java
index 55a527cc792..9bb17abf0cd 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerFactory.java
@@ -36,7 +36,7 @@ public class EdgeNGramTokenizerFactory extends AbstractTokenizerFactory {
     private final CharMatcher matcher;
 
     EdgeNGramTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         this.minGram = settings.getAsInt("min_gram", NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE);
         this.maxGram = settings.getAsInt("max_gram", NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
         this.matcher = parseTokenChars(settings.getAsList("token_chars"));
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordTokenizerFactory.java
index abe88462cb9..e4bf2c8c4ad 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordTokenizerFactory.java
@@ -31,7 +31,7 @@ public class KeywordTokenizerFactory extends AbstractTokenizerFactory {
     private final int bufferSize;
 
     KeywordTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         bufferSize = settings.getAsInt("buffer_size", 256);
     }
 
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LetterTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LetterTokenizerFactory.java
index be98eb73a9c..cba30cb63c3 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LetterTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LetterTokenizerFactory.java
@@ -29,7 +29,7 @@ import org.elasticsearch.index.analysis.AbstractTokenizerFactory;
 public class LetterTokenizerFactory extends AbstractTokenizerFactory {
 
     LetterTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
     }
 
     @Override
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenizerFactory.java
index 8f0c5f759aa..8c913a33cfe 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenizerFactory.java
@@ -30,7 +30,7 @@ import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 public class LowerCaseTokenizerFactory extends AbstractTokenizerFactory implements MultiTermAwareComponent {
 
     LowerCaseTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
     }
 
     @Override
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenizerFactory.java
index b67f67cb2fa..b00797428b7 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/NGramTokenizerFactory.java
@@ -85,7 +85,7 @@ public class NGramTokenizerFactory extends AbstractTokenizerFactory {
     }
 
     NGramTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         int maxAllowedNgramDiff = indexSettings.getMaxNgramDiff();
         this.minGram = settings.getAsInt("min_gram", NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE);
         this.maxGram = settings.getAsInt("max_gram", NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactory.java
index c877fe6944e..5b966c1c3b8 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PathHierarchyTokenizerFactory.java
@@ -37,7 +37,7 @@ public class PathHierarchyTokenizerFactory extends AbstractTokenizerFactory {
     private final boolean reverse;
 
     PathHierarchyTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         bufferSize = settings.getAsInt("buffer_size", 1024);
         String delimiter = settings.get("delimiter");
         if (delimiter == null) {
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java
index f850b68ac98..11ba7e44db0 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternTokenizerFactory.java
@@ -35,7 +35,7 @@ public class PatternTokenizerFactory extends AbstractTokenizerFactory {
     private final int group;
 
     PatternTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
 
         String sPattern = settings.get("pattern", "\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/);
         if (sPattern == null) {
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternSplitTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternSplitTokenizerFactory.java
index f861ec3792f..0faf4078295 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternSplitTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternSplitTokenizerFactory.java
@@ -31,7 +31,7 @@ public class SimplePatternSplitTokenizerFactory extends AbstractTokenizerFactory
     private final String pattern;
 
     public SimplePatternSplitTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
 
         pattern = settings.get("pattern", "");
     }
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternTokenizerFactory.java
index 6db3cfa67a3..67aee333d0f 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SimplePatternTokenizerFactory.java
@@ -31,7 +31,7 @@ public class SimplePatternTokenizerFactory extends AbstractTokenizerFactory {
     private final String pattern;
 
     public SimplePatternTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
 
         pattern = settings.get("pattern", "");
     }
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiTokenizerFactory.java
index b76aca42d36..861ade079a0 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ThaiTokenizerFactory.java
@@ -32,7 +32,7 @@ import org.elasticsearch.index.analysis.AbstractTokenizerFactory;
 public class ThaiTokenizerFactory extends AbstractTokenizerFactory {
 
     ThaiTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
     }
 
     @Override
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java
index 8040c88ea7f..cd02eec24b4 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java
@@ -32,7 +32,7 @@ public class UAX29URLEmailTokenizerFactory extends AbstractTokenizerFactory {
     private final int maxTokenLength;
 
     UAX29URLEmailTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         maxTokenLength = settings.getAsInt("max_token_length", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
     }
 
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactory.java
index 1f89d468813..7ce6a361cba 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WhitespaceTokenizerFactory.java
@@ -34,7 +34,7 @@ public class WhitespaceTokenizerFactory extends AbstractTokenizerFactory {
     private Integer maxTokenLength;
 
     WhitespaceTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         maxTokenLength = settings.getAsInt(MAX_TOKEN_LENGTH, StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
     }
 
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java
index 84c611c0f81..3f8b9296aa0 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java
@@ -47,7 +47,7 @@ public class IcuTokenizerFactory extends AbstractTokenizerFactory {
     private static final String RULE_FILES = "rule_files";
 
     public IcuTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         config = getIcuConfig(environment, settings);
     }
 
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
index 2f00e68a75e..e9268f73065 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
@@ -45,7 +45,7 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
     private boolean discartPunctuation;
 
     public KuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         mode = getMode(settings);
         userDictionary = getUserDictionary(env, settings);
         discartPunctuation = settings.getAsBoolean("discard_punctuation", true);
diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
index 346cc84e5e6..9295ed95c3f 100644
--- a/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
+++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java
@@ -38,7 +38,7 @@ public class NoriTokenizerFactory extends AbstractTokenizerFactory {
     private final KoreanTokenizer.DecompoundMode decompoundMode;
 
     public NoriTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         decompoundMode = getMode(settings);
         userDictionary = getUserDictionary(env, settings);
     }
diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseTokenizerTokenizerFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseTokenizerTokenizerFactory.java
index 9d387296152..560bce9db27 100644
--- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseTokenizerTokenizerFactory.java
+++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseTokenizerTokenizerFactory.java
@@ -28,7 +28,7 @@ import org.elasticsearch.index.IndexSettings;
 public class SmartChineseTokenizerTokenizerFactory extends AbstractTokenizerFactory {
 
     public SmartChineseTokenizerTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java
index bf6b2fd7c5b..4df0375f31c 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/AbstractTokenizerFactory.java
@@ -27,8 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 public abstract class AbstractTokenizerFactory extends AbstractIndexComponent implements TokenizerFactory {
     protected final Version version;
 
-    // TODO drop `String ignored` in a followup
-    public AbstractTokenizerFactory(IndexSettings indexSettings, String ignored, Settings settings) {
+    public AbstractTokenizerFactory(IndexSettings indexSettings, Settings settings) {
         super(indexSettings);
         this.version = Analysis.parseAnalysisVersion(this.indexSettings.getSettings(), settings, logger);
     }
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/StandardTokenizerFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/StandardTokenizerFactory.java
index ed8d2b452c2..2e4473f3b0e 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/StandardTokenizerFactory.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/StandardTokenizerFactory.java
@@ -31,7 +31,7 @@ public class StandardTokenizerFactory extends AbstractTokenizerFactory {
     private final int maxTokenLength;
 
     public StandardTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
         maxTokenLength = settings.getAsInt("max_token_length", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/TokenizerFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/TokenizerFactory.java
index be96dbd6560..4abed5a62ce 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/TokenizerFactory.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/TokenizerFactory.java
@@ -21,6 +21,6 @@ package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.Tokenizer;
 
-public interface TokenizerFactory { // TODO replace with Supplier<Tokenizer>
+public interface TokenizerFactory {
     Tokenizer create();
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizerFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizerFactory.java
index 40fee1f40f1..95cba4f2dcc 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizerFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/MlClassicTokenizerFactory.java
@@ -20,7 +20,7 @@ import org.elasticsearch.index.analysis.AbstractTokenizerFactory;
 public class MlClassicTokenizerFactory extends AbstractTokenizerFactory {
 
     public MlClassicTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
-        super(indexSettings, name, settings);
+        super(indexSettings, settings);
     }
 
     @Override

From ef81c1df57587d6e44dd750ad36926a5c8bce894 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Tue, 17 Jul 2018 09:41:34 -0400
Subject: [PATCH 061/260] Ensure to release translog snapshot in
 primary-replica resync (#32045)

Previously we create a translog snapshot inside the resync method,
and that snapshot will be closed by the resync listener. However, if
the resync method throws an exception before the resync listener
is initialized, the translog snapshot won't be released.

Closes #32030
---
 .../index/shard/PrimaryReplicaSyncer.java     | 71 ++++++++++---------
 .../shard/PrimaryReplicaSyncerTests.java      | 28 +++++---
 .../index/translog/TranslogTests.java         | 18 +++++
 3 files changed, 75 insertions(+), 42 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
index b39ebd51f2b..e66d78f2e1a 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
@@ -35,6 +35,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.tasks.Task;
@@ -80,48 +81,25 @@ public class PrimaryReplicaSyncer extends AbstractComponent {
     }
 
     public void resync(final IndexShard indexShard, final ActionListener<ResyncTask> listener) {
-        ActionListener<ResyncTask> resyncListener = null;
+        Translog.Snapshot snapshot = null;
         try {
             final long startingSeqNo = indexShard.getGlobalCheckpoint() + 1;
-            Translog.Snapshot snapshot = indexShard.newTranslogSnapshotFromMinSeqNo(startingSeqNo);
             final long maxSeqNo = indexShard.seqNoStats().getMaxSeqNo();
-            resyncListener = new ActionListener<ResyncTask>() {
-                @Override
-                public void onResponse(final ResyncTask resyncTask) {
-                    try {
-                        snapshot.close();
-                        listener.onResponse(resyncTask);
-                    } catch (final Exception e) {
-                        onFailure(e);
-                    }
-                }
-
-                @Override
-                public void onFailure(final Exception e) {
-                    try {
-                        snapshot.close();
-                    } catch (final Exception inner) {
-                        e.addSuppressed(inner);
-                    } finally {
-                        listener.onFailure(e);
-                    }
-                }
-            };
-            ShardId shardId = indexShard.shardId();
-
+            final ShardId shardId = indexShard.shardId();
             // Wrap translog snapshot to make it synchronized as it is accessed by different threads through SnapshotSender.
             // Even though those calls are not concurrent, snapshot.next() uses non-synchronized state and is not multi-thread-compatible
             // Also fail the resync early if the shard is shutting down
-            Translog.Snapshot wrappedSnapshot = new Translog.Snapshot() {
-
+            snapshot = indexShard.newTranslogSnapshotFromMinSeqNo(startingSeqNo);
+            final Translog.Snapshot originalSnapshot = snapshot;
+            final Translog.Snapshot wrappedSnapshot = new Translog.Snapshot() {
                 @Override
                 public synchronized void close() throws IOException {
-                    snapshot.close();
+                    originalSnapshot.close();
                 }
 
                 @Override
                 public synchronized int totalOperations() {
-                    return snapshot.totalOperations();
+                    return originalSnapshot.totalOperations();
                 }
 
                 @Override
@@ -132,15 +110,40 @@ public class PrimaryReplicaSyncer extends AbstractComponent {
                     } else {
                         assert state == IndexShardState.STARTED : "resync should only happen on a started shard, but state was: " + state;
                     }
-                    return snapshot.next();
+                    return originalSnapshot.next();
                 }
             };
+            final ActionListener<ResyncTask> resyncListener = new ActionListener<ResyncTask>() {
+                @Override
+                public void onResponse(final ResyncTask resyncTask) {
+                    try {
+                        wrappedSnapshot.close();
+                        listener.onResponse(resyncTask);
+                    } catch (final Exception e) {
+                        onFailure(e);
+                    }
+                }
+
+                @Override
+                public void onFailure(final Exception e) {
+                    try {
+                        wrappedSnapshot.close();
+                    } catch (final Exception inner) {
+                        e.addSuppressed(inner);
+                    } finally {
+                        listener.onFailure(e);
+                    }
+                }
+            };
+
             resync(shardId, indexShard.routingEntry().allocationId().getId(), indexShard.getPrimaryTerm(), wrappedSnapshot,
                 startingSeqNo, maxSeqNo, resyncListener);
         } catch (Exception e) {
-            if (resyncListener != null) {
-                resyncListener.onFailure(e);
-            } else {
+            try {
+                IOUtils.close(snapshot);
+            } catch (IOException inner) {
+                e.addSuppressed(inner);
+            } finally {
                 listener.onFailure(e);
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
index b290f4d4559..4444f475329 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
@@ -123,12 +123,10 @@ public class PrimaryReplicaSyncerTests extends IndexShardTestCase {
     public void testSyncerOnClosingShard() throws Exception {
         IndexShard shard = newStartedShard(true);
         AtomicBoolean syncActionCalled = new AtomicBoolean();
-        CountDownLatch syncCalledLatch = new CountDownLatch(1);
         PrimaryReplicaSyncer.SyncAction syncAction =
             (request, parentTask, allocationId, primaryTerm, listener) -> {
                 logger.info("Sending off {} operations", request.getOperations().length);
                 syncActionCalled.set(true);
-                syncCalledLatch.countDown();
                 threadPool.generic().execute(() -> listener.onResponse(new ResyncReplicationResponse()));
             };
         PrimaryReplicaSyncer syncer = new PrimaryReplicaSyncer(Settings.EMPTY,
@@ -147,13 +145,27 @@ public class PrimaryReplicaSyncerTests extends IndexShardTestCase {
         shard.updateShardState(shard.routingEntry(), shard.getPrimaryTerm(), null, 1000L, Collections.singleton(allocationId),
             new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build(), Collections.emptySet());
 
-        PlainActionFuture<PrimaryReplicaSyncer.ResyncTask> fut = new PlainActionFuture<>();
-        threadPool.generic().execute(() -> {
-            try {
-                syncer.resync(shard, fut);
-            } catch (AlreadyClosedException ace) {
-                fut.onFailure(ace);
+        CountDownLatch syncCalledLatch = new CountDownLatch(1);
+        PlainActionFuture<PrimaryReplicaSyncer.ResyncTask> fut = new PlainActionFuture<PrimaryReplicaSyncer.ResyncTask>() {
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    super.onFailure(e);
+                } finally {
+                    syncCalledLatch.countDown();
+                }
             }
+            @Override
+            public void onResponse(PrimaryReplicaSyncer.ResyncTask result) {
+                try {
+                    super.onResponse(result);
+                } finally {
+                    syncCalledLatch.countDown();
+                }
+            }
+        };
+        threadPool.generic().execute(() -> {
+            syncer.resync(shard, fut);
         });
         if (randomBoolean()) {
             syncCalledLatch.await();
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index cf6e7536846..dbbb38090bc 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -2932,6 +2932,24 @@ public class TranslogTests extends ESTestCase {
         }
     }
 
+    /** Make sure that it's ok to close a translog snapshot multiple times */
+    public void testCloseSnapshotTwice() throws Exception {
+        int numOps = between(0, 10);
+        for (int i = 0; i < numOps; i++) {
+            Translog.Index op = new Translog.Index("doc", randomAlphaOfLength(10), i, primaryTerm.get(), new byte[]{1});
+            translog.add(op);
+            if (randomBoolean()) {
+                translog.rollGeneration();
+            }
+        }
+        for (int i = 0; i < 5; i++) {
+            Translog.Snapshot snapshot = translog.newSnapshot();
+            assertThat(snapshot, SnapshotMatchers.size(numOps));
+            snapshot.close();
+            snapshot.close();
+        }
+    }
+
     static class SortedSnapshot implements Translog.Snapshot {
         private final Translog.Snapshot snapshot;
         private List<Translog.Operation> operations = null;

From 99c2a82c04872fe09e98aa461e744718b91ac3f1 Mon Sep 17 00:00:00 2001
From: David Roberts <dave.roberts@elastic.co>
Date: Tue, 17 Jul 2018 15:01:12 +0100
Subject: [PATCH 062/260] [ML] Move analyzer dependencies out of categorization
 config (#32123)

The ML config classes will shortly be moved to the X-Pack protocol
library to allow the ML APIs to be moved to the high level REST
client.  Dependencies on server functionality should be removed
from the config classes before this is done.

This change is entirely about moving code between packages.  It
does not add or remove any functionality or tests.
---
 .../config/CategorizationAnalyzerConfig.java  | 204 +----------------
 .../xpack/core/ml/job/config/Job.java         |  18 +-
 .../xpack/ml/job/JobManager.java              |  20 +-
 .../CategorizationAnalyzer.java               | 210 +++++++++++++++++-
 .../CategorizationAnalyzerTests.java          | 153 ++++++++++++-
 .../CategorizationAnalyzerConfigTests.java    | 158 -------------
 6 files changed, 382 insertions(+), 381 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
index 1c2808c70ff..fd0fde76e68 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
@@ -5,14 +5,8 @@
  */
 package org.elasticsearch.xpack.core.ml.job.config;
 
-import org.apache.lucene.analysis.Analyzer;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -22,15 +16,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.analysis.CharFilterFactory;
-import org.elasticsearch.index.analysis.CustomAnalyzer;
-import org.elasticsearch.index.analysis.CustomAnalyzerProvider;
-import org.elasticsearch.index.analysis.TokenFilterFactory;
-import org.elasticsearch.index.analysis.TokenizerFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction;
 import org.elasticsearch.xpack.core.ml.MlParserType;
 
@@ -42,12 +27,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-
 /**
  * Configuration for the categorization analyzer.
  *
  * The syntax is a subset of what can be supplied to the {@linkplain RestAnalyzeAction <code>_analyze</code> endpoint}.
- * To summarise, the first option is to specify the name of an out-of-the-box analyzer:
+ * To summarize, the first option is to specify the name of an out-of-the-box analyzer:
  * <code>
  *     "categorization_analyzer" : "standard"
  * </code>
@@ -66,11 +50,6 @@ import java.util.Objects;
  *         { "type" : "pattern_replace", "pattern": "^[0-9].*" }
  *     ]
  * </code>
- *
- * Unfortunately there is no easy to to reuse a subset of the <code>_analyze</code> action implementation, so much
- * of the code in this file is copied from {@link TransportAnalyzeAction}.  Unfortunately the logic required here is
- * not quite identical to that of {@link TransportAnalyzeAction}, and the required code is hard to partially reuse.
- * TODO: consider refactoring ES core to allow more reuse.
  */
 public class CategorizationAnalyzerConfig implements ToXContentFragment, Writeable {
 
@@ -350,175 +329,6 @@ public class CategorizationAnalyzerConfig implements ToXContentFragment, Writeab
         return builder;
     }
 
-    /**
-     * Convert the config to an {@link Analyzer}.  This may be a global analyzer or a newly created custom analyzer.
-     * In the case of a global analyzer the caller must NOT close it when they have finished with it.  In the case of
-     * a newly created custom analyzer the caller is responsible for closing it.
-     * @return The first tuple member is the {@link Analyzer}; the second indicates whether the caller is responsible
-     *         for closing it.
-     */
-    public Tuple<Analyzer, Boolean> toAnalyzer(AnalysisRegistry analysisRegistry, Environment environment) throws IOException {
-        if (analyzer != null) {
-            Analyzer globalAnalyzer = analysisRegistry.getAnalyzer(analyzer);
-            if (globalAnalyzer == null) {
-                throw new IllegalArgumentException("Failed to find global analyzer [" + analyzer + "]");
-            }
-            return new Tuple<>(globalAnalyzer, Boolean.FALSE);
-        } else {
-            List<CharFilterFactory> charFilterFactoryList =
-                    parseCharFilterFactories(analysisRegistry, environment);
-
-            Tuple<String, TokenizerFactory> tokenizerFactory = parseTokenizerFactory(analysisRegistry,
-                    environment);
-
-            List<TokenFilterFactory> tokenFilterFactoryList = parseTokenFilterFactories(analysisRegistry,
-                    environment, tokenizerFactory, charFilterFactoryList);
-
-            return new Tuple<>(new CustomAnalyzer(tokenizerFactory.v1(), tokenizerFactory.v2(),
-                    charFilterFactoryList.toArray(new CharFilterFactory[charFilterFactoryList.size()]),
-                    tokenFilterFactoryList.toArray(new TokenFilterFactory[tokenFilterFactoryList.size()])), Boolean.TRUE);
-        }
-    }
-
-
-    /**
-     * Get char filter factories for each configured char filter.  Each configuration
-     * element can be the name of an out-of-the-box char filter, or a custom definition.
-     */
-    private List<CharFilterFactory> parseCharFilterFactories(AnalysisRegistry analysisRegistry,
-                                                             Environment environment) throws IOException {
-        final List<CharFilterFactory> charFilterFactoryList = new ArrayList<>();
-        for (NameOrDefinition charFilter : charFilters) {
-            final CharFilterFactory charFilterFactory;
-            if (charFilter.name != null) {
-                AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory =
-                        analysisRegistry.getCharFilterProvider(charFilter.name);
-                if (charFilterFactoryFactory == null) {
-                    throw new IllegalArgumentException("Failed to find global char filter under [" + charFilter.name + "]");
-                }
-                charFilterFactory = charFilterFactoryFactory.get(environment, charFilter.name);
-            } else {
-                String charFilterTypeName = charFilter.definition.get("type");
-                if (charFilterTypeName == null) {
-                    throw new IllegalArgumentException("Missing [type] setting for char filter: " + charFilter.definition);
-                }
-                AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory =
-                        analysisRegistry.getCharFilterProvider(charFilterTypeName);
-                if (charFilterFactoryFactory == null) {
-                    throw new IllegalArgumentException("Failed to find global char filter under [" + charFilterTypeName + "]");
-                }
-                Settings settings = augmentSettings(charFilter.definition);
-                // Need to set anonymous "name" of char_filter
-                charFilterFactory = charFilterFactoryFactory.get(buildDummyIndexSettings(settings), environment,
-                        "_anonymous_charfilter", settings);
-            }
-            if (charFilterFactory == null) {
-                throw new IllegalArgumentException("Failed to find char filter [" + charFilter + "]");
-            }
-            charFilterFactoryList.add(charFilterFactory);
-        }
-        return charFilterFactoryList;
-    }
-
-    /**
-     * Get the tokenizer factory for the configured tokenizer.  The configuration
-     * can be the name of an out-of-the-box tokenizer, or a custom definition.
-     */
-    private Tuple<String, TokenizerFactory> parseTokenizerFactory(AnalysisRegistry analysisRegistry,
-                                                                  Environment environment) throws IOException {
-        final String name;
-        final TokenizerFactory tokenizerFactory;
-        if (tokenizer.name != null) {
-            name = tokenizer.name;
-            AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(name);
-            if (tokenizerFactoryFactory == null) {
-                throw new IllegalArgumentException("Failed to find global tokenizer under [" + name + "]");
-            }
-            tokenizerFactory = tokenizerFactoryFactory.get(environment, name);
-        } else {
-            String tokenizerTypeName = tokenizer.definition.get("type");
-            if (tokenizerTypeName == null) {
-                throw new IllegalArgumentException("Missing [type] setting for tokenizer: " + tokenizer.definition);
-            }
-            AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory =
-                    analysisRegistry.getTokenizerProvider(tokenizerTypeName);
-            if (tokenizerFactoryFactory == null) {
-                throw new IllegalArgumentException("Failed to find global tokenizer under [" + tokenizerTypeName + "]");
-            }
-            Settings settings = augmentSettings(tokenizer.definition);
-            // Need to set anonymous "name" of tokenizer
-            name = "_anonymous_tokenizer";
-            tokenizerFactory = tokenizerFactoryFactory.get(buildDummyIndexSettings(settings), environment, name, settings);
-        }
-        return new Tuple<>(name, tokenizerFactory);
-    }
-
-    /**
-     * Get token filter factories for each configured token filter.  Each configuration
-     * element can be the name of an out-of-the-box token filter, or a custom definition.
-     */
-    private List<TokenFilterFactory> parseTokenFilterFactories(AnalysisRegistry analysisRegistry, Environment environment,
-                                                               Tuple<String, TokenizerFactory> tokenizerFactory,
-                                                               List<CharFilterFactory> charFilterFactoryList) throws IOException {
-        final List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
-        for (NameOrDefinition tokenFilter : tokenFilters) {
-            TokenFilterFactory tokenFilterFactory;
-            if (tokenFilter.name != null) {
-                AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory;
-                tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name);
-                if (tokenFilterFactoryFactory == null) {
-                    throw new IllegalArgumentException("Failed to find global token filter under [" + tokenFilter.name + "]");
-                }
-                tokenFilterFactory = tokenFilterFactoryFactory.get(environment, tokenFilter.name);
-            } else {
-                String filterTypeName = tokenFilter.definition.get("type");
-                if (filterTypeName == null) {
-                    throw new IllegalArgumentException("Missing [type] setting for token filter: " + tokenFilter.definition);
-                }
-                AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory =
-                        analysisRegistry.getTokenFilterProvider(filterTypeName);
-                if (tokenFilterFactoryFactory == null) {
-                    throw new IllegalArgumentException("Failed to find global token filter under [" + filterTypeName + "]");
-                }
-                Settings settings = augmentSettings(tokenFilter.definition);
-                // Need to set anonymous "name" of token_filter
-                tokenFilterFactory = tokenFilterFactoryFactory.get(buildDummyIndexSettings(settings), environment,
-                        "_anonymous_tokenfilter", settings);
-                tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(),
-                        tokenizerFactory.v2(), tokenFilterFactoryList, charFilterFactoryList, environment);
-            }
-            if (tokenFilterFactory == null) {
-                throw new IllegalArgumentException("Failed to find or create token filter [" + tokenFilter + "]");
-            }
-            tokenFilterFactoryList.add(tokenFilterFactory);
-        }
-        return tokenFilterFactoryList;
-    }
-
-    /**
-     * The Elasticsearch analysis functionality is designed to work with indices.  For
-     * categorization we have to pretend we've got some index settings.
-     */
-    private IndexSettings buildDummyIndexSettings(Settings settings) {
-        IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
-        return new IndexSettings(metaData, Settings.EMPTY);
-    }
-
-    /**
-     * The behaviour of Elasticsearch analyzers can vary between versions.
-     * For categorization we'll always use the latest version of the text analysis.
-     * The other settings are just to stop classes that expect to be associated with
-     * an index from complaining.
-     */
-    private Settings augmentSettings(Settings settings) {
-        return Settings.builder().put(settings)
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-                .build();
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -609,17 +419,5 @@ public class CategorizationAnalyzerConfig implements ToXContentFragment, Writeab
             }
             return new CategorizationAnalyzerConfig(analyzer, charFilters, tokenizer, tokenFilters);
         }
-
-        /**
-         * Verify that the builder will build a valid config.  This is not done as part of the basic build
-         * because it verifies that the names of analyzers/tokenizers/filters referenced by the config are
-         * known, and the validity of these names could change over time.
-         */
-        public void verify(AnalysisRegistry analysisRegistry, Environment environment) throws IOException {
-            Tuple<Analyzer, Boolean> tuple = build().toAnalyzer(analysisRegistry, environment);
-            if (tuple.v2()) {
-                tuple.v1().close();
-            }
-        }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
index c8290521f98..560bac895fa 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
@@ -21,8 +21,6 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
@@ -809,8 +807,8 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
             return this;
         }
 
-        public AnalysisLimits getAnalysisLimits() {
-             return analysisLimits;
+        public AnalysisConfig getAnalysisConfig() {
+             return analysisConfig;
         }
 
         public Builder setAnalysisLimits(AnalysisLimits analysisLimits) {
@@ -1135,18 +1133,6 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
                     AnalysisLimits.DEFAULT_MODEL_MEMORY_LIMIT_MB);
         }
 
-        /**
-         * Validate the char filter/tokenizer/token filter names used in the categorization analyzer config (if any).
-         * The overall structure can be validated at parse time, but the exact names need to be checked separately,
-         * as plugins that provide the functionality can be installed/uninstalled.
-         */
-        public void validateCategorizationAnalyzer(AnalysisRegistry analysisRegistry, Environment environment) throws IOException {
-            CategorizationAnalyzerConfig categorizationAnalyzerConfig = analysisConfig.getCategorizationAnalyzerConfig();
-            if (categorizationAnalyzerConfig != null) {
-                new CategorizationAnalyzerConfig.Builder(categorizationAnalyzerConfig).verify(analysisRegistry, environment);
-            }
-        }
-
         private void validateGroups() {
             for (String group : this.groups) {
                 if (MlStrings.isValidId(group) == false) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
index c3d31ae10e9..f7fab029c88 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
@@ -39,6 +39,7 @@ import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateJobAction;
 import org.elasticsearch.xpack.core.ml.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
+import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
@@ -50,6 +51,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeSta
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzer;
 import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
 import org.elasticsearch.xpack.ml.job.process.autodetect.UpdateParams;
@@ -170,6 +172,22 @@ public class JobManager extends AbstractComponent {
         return MlMetadata.getJobState(jobId, tasks);
     }
 
+    /**
+     * Validate the char filter/tokenizer/token filter names used in the categorization analyzer config (if any).
+     * This validation has to be done server-side; it cannot be done in a client as that won't have loaded the
+     * appropriate analysis modules/plugins.
+     * The overall structure can be validated at parse time, but the exact names need to be checked separately,
+     * as plugins that provide the functionality can be installed/uninstalled.
+     */
+    static void validateCategorizationAnalyzer(Job.Builder jobBuilder, AnalysisRegistry analysisRegistry, Environment environment)
+        throws IOException {
+        CategorizationAnalyzerConfig categorizationAnalyzerConfig = jobBuilder.getAnalysisConfig().getCategorizationAnalyzerConfig();
+        if (categorizationAnalyzerConfig != null) {
+            CategorizationAnalyzer.verifyConfigBuilder(new CategorizationAnalyzerConfig.Builder(categorizationAnalyzerConfig),
+                analysisRegistry, environment);
+        }
+    }
+
     /**
      * Stores a job in the cluster state
      */
@@ -177,7 +195,7 @@ public class JobManager extends AbstractComponent {
                        ActionListener<PutJobAction.Response> actionListener) throws IOException {
 
         request.getJobBuilder().validateAnalysisLimitsAndSetDefaults(maxModelMemoryLimit);
-        request.getJobBuilder().validateCategorizationAnalyzer(analysisRegistry, environment);
+        validateCategorizationAnalyzer(request.getJobBuilder(), analysisRegistry, environment);
 
         Job job = request.getJobBuilder().build(new Date());
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java
index a0101b999d5..6111fa139f9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java
@@ -9,9 +9,21 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.CustomAnalyzer;
+import org.elasticsearch.index.analysis.CustomAnalyzerProvider;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
 
 import java.io.Closeable;
@@ -19,11 +31,16 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-
 /**
  * The categorization analyzer.
  *
  * Converts messages to lists of tokens that will be fed to the ML categorization algorithm.
+ *
+ * The code in {@link #makeAnalyzer} and the methods it calls is largely copied from {@link TransportAnalyzeAction}.
+ * Unfortunately there is no easy way to reuse a subset of the <code>_analyze</code> action implementation, as the
+ * logic required here is not quite identical to that of {@link TransportAnalyzeAction}, and the required code is
+ * hard to partially reuse.
+ * TODO: consider refactoring ES core to allow more reuse.
  */
 public class CategorizationAnalyzer implements Closeable {
 
@@ -33,7 +50,7 @@ public class CategorizationAnalyzer implements Closeable {
     public CategorizationAnalyzer(AnalysisRegistry analysisRegistry, Environment environment,
                                   CategorizationAnalyzerConfig categorizationAnalyzerConfig) throws IOException {
 
-        Tuple<Analyzer, Boolean> tuple = categorizationAnalyzerConfig.toAnalyzer(analysisRegistry, environment);
+        Tuple<Analyzer, Boolean> tuple = makeAnalyzer(categorizationAnalyzerConfig, analysisRegistry, environment);
         analyzer = tuple.v1();
         closeAnalyzer = tuple.v2();
     }
@@ -69,4 +86,193 @@ public class CategorizationAnalyzer implements Closeable {
         }
         return tokens;
     }
+
+    /**
+     * Verify that the config builder will build a valid config.  This is not done as part of the basic build
+     * because it verifies that the names of analyzers/tokenizers/filters referenced by the config are
+     * known, and the validity of these names could change over time.  Additionally, it has to be done
+     * server-side rather than client-side, as the client will not have loaded the appropriate analysis
+     * modules/plugins.
+     */
+    public static void verifyConfigBuilder(CategorizationAnalyzerConfig.Builder configBuilder, AnalysisRegistry analysisRegistry,
+                                           Environment environment) throws IOException {
+        Tuple<Analyzer, Boolean> tuple = makeAnalyzer(configBuilder.build(), analysisRegistry, environment);
+        if (tuple.v2()) {
+            tuple.v1().close();
+        }
+    }
+
+    /**
+     * Convert a config to an {@link Analyzer}.  This may be a global analyzer or a newly created custom analyzer.
+     * In the case of a global analyzer the caller must NOT close it when they have finished with it.  In the case of
+     * a newly created custom analyzer the caller is responsible for closing it.
+     * @return The first tuple member is the {@link Analyzer}; the second indicates whether the caller is responsible
+     *         for closing it.
+     */
+    private static Tuple<Analyzer, Boolean> makeAnalyzer(CategorizationAnalyzerConfig config, AnalysisRegistry analysisRegistry,
+                                                         Environment environment) throws IOException {
+        String analyzer = config.getAnalyzer();
+        if (analyzer != null) {
+            Analyzer globalAnalyzer = analysisRegistry.getAnalyzer(analyzer);
+            if (globalAnalyzer == null) {
+                throw new IllegalArgumentException("Failed to find global analyzer [" + analyzer + "]");
+            }
+            return new Tuple<>(globalAnalyzer, Boolean.FALSE);
+        } else {
+            List<CharFilterFactory> charFilterFactoryList = parseCharFilterFactories(config, analysisRegistry, environment);
+
+            Tuple<String, TokenizerFactory> tokenizerFactory = parseTokenizerFactory(config, analysisRegistry, environment);
+
+            List<TokenFilterFactory> tokenFilterFactoryList = parseTokenFilterFactories(config, analysisRegistry, environment,
+                tokenizerFactory, charFilterFactoryList);
+
+            return new Tuple<>(new CustomAnalyzer(tokenizerFactory.v1(), tokenizerFactory.v2(),
+                charFilterFactoryList.toArray(new CharFilterFactory[charFilterFactoryList.size()]),
+                tokenFilterFactoryList.toArray(new TokenFilterFactory[tokenFilterFactoryList.size()])), Boolean.TRUE);
+        }
+    }
+
+
+    /**
+     * Get char filter factories for each configured char filter.  Each configuration
+     * element can be the name of an out-of-the-box char filter, or a custom definition.
+     */
+    private static List<CharFilterFactory> parseCharFilterFactories(CategorizationAnalyzerConfig config, AnalysisRegistry analysisRegistry,
+                                                                    Environment environment) throws IOException {
+        List<CategorizationAnalyzerConfig.NameOrDefinition> charFilters = config.getCharFilters();
+        final List<CharFilterFactory> charFilterFactoryList = new ArrayList<>();
+        for (CategorizationAnalyzerConfig.NameOrDefinition charFilter : charFilters) {
+            final CharFilterFactory charFilterFactory;
+            if (charFilter.name != null) {
+                AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory =
+                    analysisRegistry.getCharFilterProvider(charFilter.name);
+                if (charFilterFactoryFactory == null) {
+                    throw new IllegalArgumentException("Failed to find global char filter under [" + charFilter.name + "]");
+                }
+                charFilterFactory = charFilterFactoryFactory.get(environment, charFilter.name);
+            } else {
+                String charFilterTypeName = charFilter.definition.get("type");
+                if (charFilterTypeName == null) {
+                    throw new IllegalArgumentException("Missing [type] setting for char filter: " + charFilter.definition);
+                }
+                AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory =
+                    analysisRegistry.getCharFilterProvider(charFilterTypeName);
+                if (charFilterFactoryFactory == null) {
+                    throw new IllegalArgumentException("Failed to find global char filter under [" + charFilterTypeName + "]");
+                }
+                Settings settings = augmentSettings(charFilter.definition);
+                // Need to set anonymous "name" of char_filter
+                charFilterFactory = charFilterFactoryFactory.get(buildDummyIndexSettings(settings), environment, "_anonymous_charfilter",
+                    settings);
+            }
+            if (charFilterFactory == null) {
+                throw new IllegalArgumentException("Failed to find char filter [" + charFilter + "]");
+            }
+            charFilterFactoryList.add(charFilterFactory);
+        }
+        return charFilterFactoryList;
+    }
+
+    /**
+     * Get the tokenizer factory for the configured tokenizer.  The configuration
+     * can be the name of an out-of-the-box tokenizer, or a custom definition.
+     */
+    private static Tuple<String, TokenizerFactory> parseTokenizerFactory(CategorizationAnalyzerConfig config,
+                                                                         AnalysisRegistry analysisRegistry, Environment environment)
+        throws IOException {
+        CategorizationAnalyzerConfig.NameOrDefinition tokenizer = config.getTokenizer();
+        final String name;
+        final TokenizerFactory tokenizerFactory;
+        if (tokenizer.name != null) {
+            name = tokenizer.name;
+            AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(name);
+            if (tokenizerFactoryFactory == null) {
+                throw new IllegalArgumentException("Failed to find global tokenizer under [" + name + "]");
+            }
+            tokenizerFactory = tokenizerFactoryFactory.get(environment, name);
+        } else {
+            String tokenizerTypeName = tokenizer.definition.get("type");
+            if (tokenizerTypeName == null) {
+                throw new IllegalArgumentException("Missing [type] setting for tokenizer: " + tokenizer.definition);
+            }
+            AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory =
+                analysisRegistry.getTokenizerProvider(tokenizerTypeName);
+            if (tokenizerFactoryFactory == null) {
+                throw new IllegalArgumentException("Failed to find global tokenizer under [" + tokenizerTypeName + "]");
+            }
+            Settings settings = augmentSettings(tokenizer.definition);
+            // Need to set anonymous "name" of tokenizer
+            name = "_anonymous_tokenizer";
+            tokenizerFactory = tokenizerFactoryFactory.get(buildDummyIndexSettings(settings), environment, name, settings);
+        }
+        return new Tuple<>(name, tokenizerFactory);
+    }
+
+    /**
+     * Get token filter factories for each configured token filter.  Each configuration
+     * element can be the name of an out-of-the-box token filter, or a custom definition.
+     */
+    private static List<TokenFilterFactory> parseTokenFilterFactories(CategorizationAnalyzerConfig config,
+                                                                      AnalysisRegistry analysisRegistry, Environment environment,
+                                                                      Tuple<String, TokenizerFactory> tokenizerFactory,
+                                                                      List<CharFilterFactory> charFilterFactoryList) throws IOException {
+        List<CategorizationAnalyzerConfig.NameOrDefinition> tokenFilters = config.getTokenFilters();
+        final List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
+        for (CategorizationAnalyzerConfig.NameOrDefinition tokenFilter : tokenFilters) {
+            TokenFilterFactory tokenFilterFactory;
+            if (tokenFilter.name != null) {
+                AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory;
+                tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name);
+                if (tokenFilterFactoryFactory == null) {
+                    throw new IllegalArgumentException("Failed to find global token filter under [" + tokenFilter.name + "]");
+                }
+                tokenFilterFactory = tokenFilterFactoryFactory.get(environment, tokenFilter.name);
+            } else {
+                String filterTypeName = tokenFilter.definition.get("type");
+                if (filterTypeName == null) {
+                    throw new IllegalArgumentException("Missing [type] setting for token filter: " + tokenFilter.definition);
+                }
+                AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory =
+                    analysisRegistry.getTokenFilterProvider(filterTypeName);
+                if (tokenFilterFactoryFactory == null) {
+                    throw new IllegalArgumentException("Failed to find global token filter under [" + filterTypeName + "]");
+                }
+                Settings settings = augmentSettings(tokenFilter.definition);
+                // Need to set anonymous "name" of token_filter
+                tokenFilterFactory = tokenFilterFactoryFactory.get(buildDummyIndexSettings(settings), environment, "_anonymous_tokenfilter",
+                    settings);
+                tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(),
+                    tokenizerFactory.v2(), tokenFilterFactoryList, charFilterFactoryList, environment);
+            }
+            if (tokenFilterFactory == null) {
+                throw new IllegalArgumentException("Failed to find or create token filter [" + tokenFilter + "]");
+            }
+            tokenFilterFactoryList.add(tokenFilterFactory);
+        }
+        return tokenFilterFactoryList;
+    }
+
+    /**
+     * The Elasticsearch analysis functionality is designed to work with indices.  For
+     * categorization we have to pretend we've got some index settings.
+     */
+    private static IndexSettings buildDummyIndexSettings(Settings settings) {
+        IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
+        return new IndexSettings(metaData, Settings.EMPTY);
+    }
+
+    /**
+     * The behaviour of Elasticsearch analyzers can vary between versions.
+     * For categorization we'll always use the latest version of the text analysis.
+     * The other settings are just to stop classes that expect to be associated with
+     * an index from complaining.
+     */
+    private static Settings augmentSettings(Settings settings) {
+        return Settings.builder().put(settings)
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .build();
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzerTests.java
index 9f03952165c..59413f6a618 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzerTests.java
@@ -22,7 +22,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-
 public class CategorizationAnalyzerTests extends ESTestCase {
 
     private AnalysisRegistry analysisRegistry;
@@ -41,6 +40,158 @@ public class CategorizationAnalyzerTests extends ESTestCase {
         analysisRegistry = buildTestAnalysisRegistry(environment);
     }
 
+    public void testVerifyConfigBuilder_GivenNoConfig() {
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("categorization_analyzer that is not a global analyzer must specify a [tokenizer] field", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenDefault() throws IOException {
+        CategorizationAnalyzerConfig defaultConfig = CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(null);
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder(defaultConfig);
+        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment);
+    }
+
+    public void testVerifyConfigBuilder_GivenValidAnalyzer() throws IOException {
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder().setAnalyzer("standard");
+        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment);
+    }
+
+    public void testVerifyConfigBuilder_GivenInvalidAnalyzer() {
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder().setAnalyzer("does not exist");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("Failed to find global analyzer [does not exist]", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenValidCustomConfig() throws IOException {
+        Map<String, Object> ignoreStuffInSqaureBrackets = new HashMap<>();
+        ignoreStuffInSqaureBrackets.put("type", "pattern_replace");
+        ignoreStuffInSqaureBrackets.put("pattern", "\\[[^\\]]*\\]");
+        Map<String, Object> ignoreStuffThatBeginsWithADigit = new HashMap<>();
+        ignoreStuffThatBeginsWithADigit.put("type", "pattern_replace");
+        ignoreStuffThatBeginsWithADigit.put("pattern", "^[0-9].*");
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .addCharFilter(ignoreStuffInSqaureBrackets)
+            .setTokenizer("classic")
+            .addTokenFilter("lowercase")
+            .addTokenFilter(ignoreStuffThatBeginsWithADigit)
+            .addTokenFilter("snowball");
+        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment);
+    }
+
+    public void testVerifyConfigBuilder_GivenCustomConfigWithInvalidCharFilter() {
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .addCharFilter("wrong!")
+            .setTokenizer("classic")
+            .addTokenFilter("lowercase")
+            .addTokenFilter("snowball");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("Failed to find global char filter under [wrong!]", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenCustomConfigWithMisconfiguredCharFilter() {
+        Map<String, Object> noPattern = new HashMap<>();
+        noPattern.put("type", "pattern_replace");
+        noPattern.put("attern", "should have been pattern");
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .addCharFilter(noPattern)
+            .setTokenizer("classic")
+            .addTokenFilter("lowercase")
+            .addTokenFilter("snowball");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("pattern is missing for [_anonymous_charfilter] char filter of type 'pattern_replace'", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenCustomConfigWithInvalidTokenizer() {
+        Map<String, Object> ignoreStuffInSqaureBrackets = new HashMap<>();
+        ignoreStuffInSqaureBrackets.put("type", "pattern_replace");
+        ignoreStuffInSqaureBrackets.put("pattern", "\\[[^\\]]*\\]");
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .addCharFilter(ignoreStuffInSqaureBrackets)
+            .setTokenizer("oops!")
+            .addTokenFilter("lowercase")
+            .addTokenFilter("snowball");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("Failed to find global tokenizer under [oops!]", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenNoTokenizer() {
+        Map<String, Object> ignoreStuffInSqaureBrackets = new HashMap<>();
+        ignoreStuffInSqaureBrackets.put("type", "pattern_replace");
+        ignoreStuffInSqaureBrackets.put("pattern", "\\[[^\\]]*\\]");
+        Map<String, Object> ignoreStuffThatBeginsWithADigit = new HashMap<>();
+        ignoreStuffThatBeginsWithADigit.put("type", "pattern_replace");
+        ignoreStuffThatBeginsWithADigit.put("pattern", "^[0-9].*");
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .addCharFilter(ignoreStuffInSqaureBrackets)
+            .addTokenFilter("lowercase")
+            .addTokenFilter(ignoreStuffThatBeginsWithADigit)
+            .addTokenFilter("snowball");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("categorization_analyzer that is not a global analyzer must specify a [tokenizer] field", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenCustomConfigWithInvalidTokenFilter() {
+        Map<String, Object> ignoreStuffInSqaureBrackets = new HashMap<>();
+        ignoreStuffInSqaureBrackets.put("type", "pattern_replace");
+        ignoreStuffInSqaureBrackets.put("pattern", "\\[[^\\]]*\\]");
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .addCharFilter(ignoreStuffInSqaureBrackets)
+            .setTokenizer("classic")
+            .addTokenFilter("lowercase")
+            .addTokenFilter("oh dear!");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("Failed to find global token filter under [oh dear!]", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenCustomConfigWithMisconfiguredTokenFilter() {
+        Map<String, Object> noPattern = new HashMap<>();
+        noPattern.put("type", "pattern_replace");
+        noPattern.put("attern", "should have been pattern");
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .addCharFilter("html_strip")
+            .setTokenizer("classic")
+            .addTokenFilter("lowercase")
+            .addTokenFilter(noPattern);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("pattern is missing for [_anonymous_tokenfilter] token filter of type 'pattern_replace'", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenAnalyzerAndCharFilter() {
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .setAnalyzer("standard")
+            .addCharFilter("html_strip");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [char_filter] field", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenAnalyzerAndTokenizer() {
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .setAnalyzer("standard")
+            .setTokenizer("classic");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [tokenizer] field", e.getMessage());
+    }
+
+    public void testVerifyConfigBuilder_GivenAnalyzerAndTokenFilter() {
+        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
+            .setAnalyzer("standard")
+            .addTokenFilter("lowercase");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+        assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [filter] field", e.getMessage());
+    }
+
     // The default categorization analyzer matches what the analyzer in the ML C++ does
     public void testDefaultCategorizationAnalyzer() throws IOException {
         CategorizationAnalyzerConfig defaultConfig = CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(null);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java
index 9c725fe7629..2fe2c0b334c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java
@@ -6,175 +6,17 @@
 package org.elasticsearch.xpack.ml.job.config;
 
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
 import org.elasticsearch.xpack.core.ml.MlParserType;
-import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests;
-import org.junit.Before;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
-
 public class CategorizationAnalyzerConfigTests extends AbstractSerializingTestCase<CategorizationAnalyzerConfig> {
 
-    private AnalysisRegistry analysisRegistry;
-    private Environment environment;
-
-    @Before
-    public void setup() throws Exception {
-        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
-        environment = TestEnvironment.newEnvironment(settings);
-        analysisRegistry = CategorizationAnalyzerTests.buildTestAnalysisRegistry(environment);
-    }
-
-    public void testVerify_GivenNoConfig() {
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("categorization_analyzer that is not a global analyzer must specify a [tokenizer] field", e.getMessage());
-    }
-
-    public void testVerify_GivenDefault() throws IOException {
-        CategorizationAnalyzerConfig defaultConfig = CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(null);
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder(defaultConfig);
-        builder.verify(analysisRegistry, environment);
-    }
-
-    public void testVerify_GivenValidAnalyzer() throws IOException {
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder().setAnalyzer("standard");
-        builder.verify(analysisRegistry, environment);
-    }
-
-    public void testVerify_GivenInvalidAnalyzer() {
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder().setAnalyzer("does not exist");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("Failed to find global analyzer [does not exist]", e.getMessage());
-    }
-
-    public void testVerify_GivenValidCustomConfig() throws IOException {
-        Map<String, Object> ignoreStuffInSqaureBrackets = new HashMap<>();
-        ignoreStuffInSqaureBrackets.put("type", "pattern_replace");
-        ignoreStuffInSqaureBrackets.put("pattern", "\\[[^\\]]*\\]");
-        Map<String, Object> ignoreStuffThatBeginsWithADigit = new HashMap<>();
-        ignoreStuffThatBeginsWithADigit.put("type", "pattern_replace");
-        ignoreStuffThatBeginsWithADigit.put("pattern", "^[0-9].*");
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .addCharFilter(ignoreStuffInSqaureBrackets)
-                .setTokenizer("classic")
-                .addTokenFilter("lowercase")
-                .addTokenFilter(ignoreStuffThatBeginsWithADigit)
-                .addTokenFilter("snowball");
-        builder.verify(analysisRegistry, environment);
-    }
-
-    public void testVerify_GivenCustomConfigWithInvalidCharFilter() {
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .addCharFilter("wrong!")
-                .setTokenizer("classic")
-                .addTokenFilter("lowercase")
-                .addTokenFilter("snowball");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("Failed to find global char filter under [wrong!]", e.getMessage());
-    }
-
-    public void testVerify_GivenCustomConfigWithMisconfiguredCharFilter() {
-        Map<String, Object> noPattern = new HashMap<>();
-        noPattern.put("type", "pattern_replace");
-        noPattern.put("attern", "should have been pattern");
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .addCharFilter(noPattern)
-                .setTokenizer("classic")
-                .addTokenFilter("lowercase")
-                .addTokenFilter("snowball");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("pattern is missing for [_anonymous_charfilter] char filter of type 'pattern_replace'", e.getMessage());
-    }
-
-    public void testVerify_GivenCustomConfigWithInvalidTokenizer() {
-        Map<String, Object> ignoreStuffInSqaureBrackets = new HashMap<>();
-        ignoreStuffInSqaureBrackets.put("type", "pattern_replace");
-        ignoreStuffInSqaureBrackets.put("pattern", "\\[[^\\]]*\\]");
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .addCharFilter(ignoreStuffInSqaureBrackets)
-                .setTokenizer("oops!")
-                .addTokenFilter("lowercase")
-                .addTokenFilter("snowball");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("Failed to find global tokenizer under [oops!]", e.getMessage());
-    }
-
-    public void testVerify_GivenNoTokenizer() {
-        Map<String, Object> ignoreStuffInSqaureBrackets = new HashMap<>();
-        ignoreStuffInSqaureBrackets.put("type", "pattern_replace");
-        ignoreStuffInSqaureBrackets.put("pattern", "\\[[^\\]]*\\]");
-        Map<String, Object> ignoreStuffThatBeginsWithADigit = new HashMap<>();
-        ignoreStuffThatBeginsWithADigit.put("type", "pattern_replace");
-        ignoreStuffThatBeginsWithADigit.put("pattern", "^[0-9].*");
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .addCharFilter(ignoreStuffInSqaureBrackets)
-                .addTokenFilter("lowercase")
-                .addTokenFilter(ignoreStuffThatBeginsWithADigit)
-                .addTokenFilter("snowball");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("categorization_analyzer that is not a global analyzer must specify a [tokenizer] field", e.getMessage());
-    }
-
-    public void testVerify_GivenCustomConfigWithInvalidTokenFilter() {
-        Map<String, Object> ignoreStuffInSqaureBrackets = new HashMap<>();
-        ignoreStuffInSqaureBrackets.put("type", "pattern_replace");
-        ignoreStuffInSqaureBrackets.put("pattern", "\\[[^\\]]*\\]");
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .addCharFilter(ignoreStuffInSqaureBrackets)
-                .setTokenizer("classic")
-                .addTokenFilter("lowercase")
-                .addTokenFilter("oh dear!");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("Failed to find global token filter under [oh dear!]", e.getMessage());
-    }
-
-    public void testVerify_GivenCustomConfigWithMisconfiguredTokenFilter() {
-        Map<String, Object> noPattern = new HashMap<>();
-        noPattern.put("type", "pattern_replace");
-        noPattern.put("attern", "should have been pattern");
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .addCharFilter("html_strip")
-                .setTokenizer("classic")
-                .addTokenFilter("lowercase")
-                .addTokenFilter(noPattern);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("pattern is missing for [_anonymous_tokenfilter] token filter of type 'pattern_replace'", e.getMessage());
-    }
-
-    public void testVerify_GivenAnalyzerAndCharFilter() {
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .setAnalyzer("standard")
-                .addCharFilter("html_strip");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [char_filter] field", e.getMessage());
-    }
-
-    public void testVerify_GivenAnalyzerAndTokenizer() {
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .setAnalyzer("standard")
-                .setTokenizer("classic");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [tokenizer] field", e.getMessage());
-    }
-
-    public void testVerify_GivenAnalyzerAndTokenFilter() {
-        CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder()
-                .setAnalyzer("standard")
-                .addTokenFilter("lowercase");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.verify(analysisRegistry, environment));
-        assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [filter] field", e.getMessage());
-    }
-
     @Override
     protected CategorizationAnalyzerConfig createTestInstance() {
         return createRandomized().build();

From 2bd40a79276f1e73ebfdda8c0c28a26800b4e349 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Tue, 17 Jul 2018 16:21:53 +0100
Subject: [PATCH 063/260] [ML] Wait for aliases in multi-node tests (#32086)

---
 .../xpack/ml/integration/MlJobIT.java         | 62 ++++++++++++-------
 .../integration/RestoreModelSnapshotIT.java   | 12 ++--
 2 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
index 7820cbc06f5..07529acdb88 100644
--- a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
+++ b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ml.integration;
 
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
@@ -185,23 +186,32 @@ public class MlJobIT extends ESRestTestCase {
                         + "anomaly_detectors/" + jobId2, Collections.emptyMap(), new StringEntity(jobConfig, ContentType.APPLICATION_JSON));
         assertEquals(200, response.getStatusLine().getStatusCode());
 
-        response = client().performRequest("get", "_aliases");
-        assertEquals(200, response.getStatusLine().getStatusCode());
-        String responseAsString = responseEntityToString(response);
+        // With security enabled GET _aliases throws an index_not_found_exception
+        // if no aliases have been created. In multi-node tests the alias may not
+        // appear immediately so wait here.
+        assertBusy(() -> {
+            try {
+                Response aliasesResponse = client().performRequest("get", "_aliases");
+                assertEquals(200, aliasesResponse.getStatusLine().getStatusCode());
+                String responseAsString = responseEntityToString(aliasesResponse);
+                assertThat(responseAsString,
+                        containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName) + "\":{\"aliases\":{"));
+                assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)
+                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}}}"));
+                assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1) + "\":{}"));
+                assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)
+                        + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}}}"));
+                assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2) + "\":{}"));
+            } catch (ResponseException e) {
+                throw new AssertionError(e);
+            }
+        });
 
+        Response indicesResponse = client().performRequest("get", "_cat/indices");
+        assertEquals(200, indicesResponse.getStatusLine().getStatusCode());
+        String responseAsString = responseEntityToString(indicesResponse);
         assertThat(responseAsString,
-                containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName) + "\":{\"aliases\":{"));
-        assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)
-                + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId1 + "\",\"boost\":1.0}}}}"));
-        assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId1) + "\":{}"));
-        assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)
-                + "\":{\"filter\":{\"term\":{\"job_id\":{\"value\":\"" + jobId2 + "\",\"boost\":1.0}}}}"));
-        assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.resultsWriteAlias(jobId2) + "\":{}"));
-
-        response = client().performRequest("get", "_cat/indices");
-        assertEquals(200, response.getStatusLine().getStatusCode());
-        responseAsString = responseEntityToString(response);
-        assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
+                containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName));
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1))));
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
 
@@ -438,7 +448,6 @@ public class MlJobIT extends ESRestTestCase {
                 client().performRequest("get", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32034")
     public void testDeleteJobAfterMissingAliases() throws Exception {
         String jobId = "delete-job-after-missing-alias-job";
         String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
@@ -446,15 +455,24 @@ public class MlJobIT extends ESRestTestCase {
         String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
         createFarequoteJob(jobId);
 
-        Response response = client().performRequest("get", "_cat/aliases");
-        assertEquals(200, response.getStatusLine().getStatusCode());
-        String responseAsString = responseEntityToString(response);
-        assertThat(responseAsString, containsString(readAliasName));
-        assertThat(responseAsString, containsString(writeAliasName));
+        // With security enabled cat aliases throws an index_not_found_exception
+        // if no aliases have been created. In multi-node tests the alias may not
+        // appear immediately so wait here.
+        assertBusy(() -> {
+            try {
+                Response aliasesResponse = client().performRequest(new Request("get", "_cat/aliases"));
+                assertEquals(200, aliasesResponse.getStatusLine().getStatusCode());
+                String responseAsString = responseEntityToString(aliasesResponse);
+                assertThat(responseAsString, containsString(readAliasName));
+                assertThat(responseAsString, containsString(writeAliasName));
+            } catch (ResponseException e) {
+                throw new AssertionError(e);
+            }
+        });
 
         // Manually delete the aliases so that we can test that deletion proceeds
         // normally anyway
-        response = client().performRequest("delete", indexName + "/_alias/" + readAliasName);
+        Response response = client().performRequest("delete", indexName + "/_alias/" + readAliasName);
         assertEquals(200, response.getStatusLine().getStatusCode());
         response = client().performRequest("delete", indexName + "/_alias/" + writeAliasName);
         assertEquals(200, response.getStatusLine().getStatusCode());
diff --git a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java
index 9ff80bc739b..d7a2b857bf3 100644
--- a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java
+++ b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java
@@ -72,10 +72,14 @@ public class RestoreModelSnapshotIT extends MlNativeAutodetectIntegTestCase {
         openJob(job.getId());
         String forecastId = forecast(job.getId(), TimeValue.timeValueHours(3), null);
         waitForecastToFinish(job.getId(), forecastId);
-        ForecastRequestStats forecastStats = getForecastStats(job.getId(), forecastId);
-        assertThat(forecastStats.getMessages(), anyOf(nullValue(), empty()));
-        assertThat(forecastStats.getMemoryUsage(), greaterThan(0L));
-        assertEquals(forecastStats.getRecordCount(), 3L);
+        // In a multi-node cluster the replica may not be up to date
+        // so wait for the change
+        assertBusy(() -> {
+            ForecastRequestStats forecastStats = getForecastStats(job.getId(), forecastId);
+            assertThat(forecastStats.getMessages(), anyOf(nullValue(), empty()));
+            assertThat(forecastStats.getMemoryUsage(), greaterThan(0L));
+            assertThat(forecastStats.getRecordCount(), equalTo(3L));
+        });
 
         closeJob(job.getId());
 

From efb4e97cfb5dc35da640d8e2b8d249b11db54c1e Mon Sep 17 00:00:00 2001
From: aptxx <prestorpan@gmail.com>
Date: Tue, 17 Jul 2018 23:42:24 +0800
Subject: [PATCH 064/260] Docs: Fix missing example script quote (#32010)

---
 .../client/documentation/ReindexDocumentationIT.java            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
index 0766560a849..93c785e754a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
@@ -96,7 +96,7 @@ public class ReindexDocumentationIT extends ESIntegTestCase {
             updateByQuery.source("source_index")
                 .script(new Script(
                     ScriptType.INLINE,
-                    "if (ctx._source.awesome == 'absolutely) {"
+                    "if (ctx._source.awesome == 'absolutely') {"
                         + "  ctx.op='noop'"
                         + "} else if (ctx._source.awesome == 'lame') {"
                         + "  ctx.op='delete'"

From 0f7d2044c321d49c2e1ed71273d4f39fc7cfeb9a Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Tue, 17 Jul 2018 09:15:11 -0700
Subject: [PATCH 065/260] Re-disable packaging tests on suse boxes

This reverts commit 14d7e2c7b2d3761a361edd720f98353f856936a4.
---
 .../gradle/vagrant/VagrantTestPlugin.groovy          | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
index d4d1d857e90..de3c0dfc328 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -526,7 +526,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                     project.gradle.removeListener(batsPackagingReproListener)
                 }
                 if (project.extensions.esvagrant.boxes.contains(box)) {
-                    packagingTest.dependsOn(batsPackagingTest)
+                    // these tests are temporarily disabled for suse boxes while we debug an issue
+                    // https://github.com/elastic/elasticsearch/issues/30295
+                    if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                        packagingTest.dependsOn(batsPackagingTest)
+                    }
                 }
             }
 
@@ -565,7 +569,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                 project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                packagingTest.dependsOn(javaPackagingTest)
+                // these tests are temporarily disabled for suse boxes while we debug an issue
+                // https://github.com/elastic/elasticsearch/issues/30295
+                if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                    packagingTest.dependsOn(javaPackagingTest)
+                }
             }
 
             /*

From 9cdbd1bd85e92181ee258b9c874be985fcc61de5 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ioannis@elastic.co>
Date: Tue, 17 Jul 2018 20:25:38 +0300
Subject: [PATCH 066/260] Remove empty @param from Javadoc

---
 .../test/java/org/elasticsearch/test/SecuritySettingsSource.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
index 7d329781fad..56d5fec3f20 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
@@ -298,7 +298,6 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas
      * Returns the SSL related configuration settings given the location of a key and certificate and the location
      * of the PEM certificates to be trusted
      *
-     * @param builder
      * @param keyPath             The path to the Private key to be used for SSL
      * @param password            The password with which the private key is protected
      * @param certificatePath     The path to the PEM formatted Certificate encapsulating the public key that corresponds

From 1c63eb108164af85a4006650f9c9753f64eb5622 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 17 Jul 2018 10:33:38 -0700
Subject: [PATCH 067/260] Painless: Fix Bug with Duplicate PainlessClasses
 (#32110)

When building the PainlessMethods and PainlessFields they stored a reference to a
PainlessClass. This reference was prior to "freezing" the PainlessClass so the data was
both incomplete and mutable. This has been replaced with a target java class instead
since the PainlessClass is accessible through a java class now and it requires no special
modifications to get around a chicken and egg issue.
---
 .../java/org/elasticsearch/painless/Def.java  |  4 +-
 .../elasticsearch/painless/FunctionRef.java   |  8 ++--
 .../painless/lookup/PainlessField.java        |  6 +--
 .../lookup/PainlessLookupBuilder.java         | 14 +++----
 .../lookup/PainlessLookupUtility.java         |  6 +--
 .../painless/lookup/PainlessMethod.java       | 15 +++----
 .../painless/node/EListInit.java              |  3 +-
 .../elasticsearch/painless/node/EMapInit.java |  3 +-
 .../elasticsearch/painless/node/ENewObj.java  |  3 +-
 .../painless/node/PSubField.java              | 13 +++---
 .../painless/PainlessDocGenerator.java        | 42 ++++++++++---------
 11 files changed, 62 insertions(+), 55 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 78db712d183..fe11ff4814b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -334,8 +334,8 @@ public final class Def {
          }
          int arity = interfaceMethod.arguments.size();
          PainlessMethod implMethod = lookupMethodInternal(painlessLookup, receiverClass, name, arity);
-        return lookupReferenceInternal(painlessLookup, methodHandlesLookup, interfaceType, implMethod.owner.name,
-                implMethod.name, receiverClass);
+        return lookupReferenceInternal(painlessLookup, methodHandlesLookup, interfaceType,
+                PainlessLookupUtility.anyTypeToPainlessTypeName(implMethod.target), implMethod.name, receiverClass);
      }
 
      /** Returns a method handle to an implementation of clazz, given method reference signature. */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 9e72dc2c835..925359fabc5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -102,22 +102,22 @@ public class FunctionRef {
         interfaceMethodType = interfaceMethod.getMethodType().dropParameterTypes(0, 1);
 
         // the Painless$Script class can be inferred if owner is null
-        if (delegateMethod.owner == null) {
+        if (delegateMethod.target == null) {
             delegateClassName = CLASS_NAME;
             isDelegateInterface = false;
         } else if (delegateMethod.augmentation != null) {
             delegateClassName = delegateMethod.augmentation.getName();
             isDelegateInterface = delegateMethod.augmentation.isInterface();
         } else {
-            delegateClassName = delegateMethod.owner.clazz.getName();
-            isDelegateInterface = delegateMethod.owner.clazz.isInterface();
+            delegateClassName = delegateMethod.target.getName();
+            isDelegateInterface = delegateMethod.target.isInterface();
         }
 
         if ("<init>".equals(delegateMethod.name)) {
             delegateInvokeType = H_NEWINVOKESPECIAL;
         } else if (Modifier.isStatic(delegateMethod.modifiers)) {
             delegateInvokeType = H_INVOKESTATIC;
-        } else if (delegateMethod.owner.clazz.isInterface()) {
+        } else if (delegateMethod.target.isInterface()) {
             delegateInvokeType = H_INVOKEINTERFACE;
         } else {
             delegateInvokeType = H_INVOKEVIRTUAL;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java
index 7c85bd269b4..f316e1438ec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java
@@ -23,18 +23,18 @@ import java.lang.invoke.MethodHandle;
 
 public final class PainlessField {
     public final String name;
-    public final PainlessClass owner;
+    public final Class<?> target;
     public final Class<?> clazz;
     public final String javaName;
     public final int modifiers;
     public final MethodHandle getter;
     public final MethodHandle setter;
 
-    PainlessField(String name, String javaName, PainlessClass owner, Class<?> clazz, int modifiers,
+    PainlessField(String name, String javaName, Class<?> target, Class<?> clazz, int modifiers,
                   MethodHandle getter, MethodHandle setter) {
         this.name = name;
         this.javaName = javaName;
-        this.owner = owner;
+        this.target = target;
         this.clazz = clazz;
         this.modifiers = modifiers;
         this.getter = getter;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 9a5e08d65a7..5641eee1b5d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -310,7 +310,7 @@ public class PainlessLookupBuilder {
             }
 
             painlessConstructor = methodCache.computeIfAbsent(buildMethodCacheKey(ownerStruct.name, "<init>", painlessParametersTypes),
-                key -> new PainlessMethod("<init>", ownerStruct, null, void.class, painlessParametersTypes,
+                key -> new PainlessMethod("<init>", ownerStruct.clazz, null, void.class, painlessParametersTypes,
                     asmConstructor, javaConstructor.getModifiers(), javaHandle));
             ownerStruct.constructors.put(painlessMethodKey, painlessConstructor);
         } else if (painlessConstructor.arguments.equals(painlessParametersTypes) == false){
@@ -419,7 +419,7 @@ public class PainlessLookupBuilder {
 
                 painlessMethod = methodCache.computeIfAbsent(
                     buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes),
-                    key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct, null, painlessReturnClass,
+                    key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct.clazz, null, painlessReturnClass,
                         painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
                 ownerStruct.staticMethods.put(painlessMethodKey, painlessMethod);
             } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn == painlessReturnClass &&
@@ -445,7 +445,7 @@ public class PainlessLookupBuilder {
 
                 painlessMethod = methodCache.computeIfAbsent(
                     buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes),
-                    key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct, javaAugmentedClass, painlessReturnClass,
+                    key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct.clazz, javaAugmentedClass, painlessReturnClass,
                         painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
                 ownerStruct.methods.put(painlessMethodKey, painlessMethod);
             } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn.equals(painlessReturnClass) &&
@@ -501,7 +501,7 @@ public class PainlessLookupBuilder {
                 painlessField = fieldCache.computeIfAbsent(
                     buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()),
                     key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
-                        ownerStruct, painlessFieldClass, javaField.getModifiers(), null, null));
+                        ownerStruct.clazz, painlessFieldClass, javaField.getModifiers(), null, null));
                 ownerStruct.staticMembers.put(whitelistField.javaFieldName, painlessField);
             } else if (painlessField.clazz != painlessFieldClass) {
                 throw new IllegalArgumentException("illegal duplicate static fields [" + whitelistField.javaFieldName + "] " +
@@ -530,7 +530,7 @@ public class PainlessLookupBuilder {
                 painlessField = fieldCache.computeIfAbsent(
                     buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()),
                     key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
-                        ownerStruct, painlessFieldClass, javaField.getModifiers(), javaMethodHandleGetter, javaMethodHandleSetter));
+                        ownerStruct.clazz, painlessFieldClass, javaField.getModifiers(), javaMethodHandleGetter, javaMethodHandleSetter));
                 ownerStruct.members.put(whitelistField.javaFieldName, painlessField);
             } else if (painlessField.clazz != painlessFieldClass) {
                 throw new IllegalArgumentException("illegal duplicate member fields [" + whitelistField.javaFieldName + "] " +
@@ -615,8 +615,8 @@ public class PainlessLookupBuilder {
 
             for (PainlessField field : child.members.values()) {
                 if (owner.members.get(field.name) == null) {
-                    owner.members.put(field.name,
-                        new PainlessField(field.name, field.javaName, owner, field.clazz, field.modifiers, field.getter, field.setter));
+                    owner.members.put(field.name, new PainlessField(
+                            field.name, field.javaName, owner.clazz, field.clazz, field.modifiers, field.getter, field.setter));
                 }
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
index d1f3ee4ece3..0f7c8fb915c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
@@ -158,7 +158,7 @@ public final class PainlessLookupUtility {
                     painlessTypeName.charAt(arrayIndex++) == ']') {
                     ++arrayDimensions;
                 } else {
-                    throw new IllegalArgumentException("invalid painless type [" + painlessTypeName + "].");
+                    throw new IllegalArgumentException("painless type [" + painlessTypeName + "] not found");
                 }
             }
 
@@ -192,7 +192,7 @@ public final class PainlessLookupUtility {
             try {
                 return Class.forName(javaDescriptor);
             } catch (ClassNotFoundException cnfe) {
-                throw new IllegalStateException("painless type [" + painlessTypeName + "] not found", cnfe);
+                throw new IllegalArgumentException("painless type [" + painlessTypeName + "] not found", cnfe);
             }
         }
 
@@ -207,7 +207,7 @@ public final class PainlessLookupUtility {
         }
 
         if (javaClasses.contains(painlessType) == false) {
-            throw new IllegalStateException("painless type [" + painlessTypeName + "] not found");
+            throw new IllegalArgumentException("painless type [" + painlessTypeName + "] not found");
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
index 8d8a7f691fe..2b0d44e7176 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.lookup;
 
 import org.elasticsearch.painless.MethodWriter;
 import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodType;
@@ -30,7 +31,7 @@ import java.util.List;
 
 public class PainlessMethod {
     public final String name;
-    public final PainlessClass owner;
+    public final Class<?> target;
     public final Class<?> augmentation;
     public final Class<?> rtn;
     public final List<Class<?>> arguments;
@@ -38,11 +39,11 @@ public class PainlessMethod {
     public final int modifiers;
     public final MethodHandle handle;
 
-    public PainlessMethod(String name, PainlessClass owner, Class<?> augmentation, Class<?> rtn, List<Class<?>> arguments,
+    public PainlessMethod(String name, Class<?> target, Class<?> augmentation, Class<?> rtn, List<Class<?>> arguments,
                           org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) {
         this.name = name;
         this.augmentation = augmentation;
-        this.owner = owner;
+        this.target = target;
         this.rtn = rtn;
         this.arguments = Collections.unmodifiableList(arguments);
         this.method = method;
@@ -85,11 +86,11 @@ public class PainlessMethod {
             for (int i = 0; i < arguments.size(); i++) {
                 params[i] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
             }
-            returnValue = owner.clazz;
+            returnValue = target;
         } else {
             // virtual/interface method: add receiver class
             params = new Class<?>[1 + arguments.size()];
-            params[0] = owner.clazz;
+            params[0] = target;
             for (int i = 0; i < arguments.size(); i++) {
                 params[i + 1] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
             }
@@ -106,8 +107,8 @@ public class PainlessMethod {
             clazz = augmentation;
             type = org.objectweb.asm.Type.getType(augmentation);
         } else {
-            clazz = owner.clazz;
-            type = owner.type;
+            clazz = target;
+            type = Type.getType(target);
         }
 
         if (Modifier.isStatic(modifiers)) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index 90475419b32..7e923e5f90f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -26,6 +26,7 @@ import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.def;
+import org.objectweb.asm.Type;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -90,7 +91,7 @@ public final class EListInit extends AExpression {
 
         writer.newInstance(MethodWriter.getType(actual));
         writer.dup();
-        writer.invokeConstructor(constructor.owner.type, constructor.method);
+        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
 
         for (AExpression value : values) {
             writer.dup();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index c6474846d4c..b350a758944 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -26,6 +26,7 @@ import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.def;
+import org.objectweb.asm.Type;
 
 import java.util.HashMap;
 import java.util.List;
@@ -109,7 +110,7 @@ public final class EMapInit extends AExpression {
 
         writer.newInstance(MethodWriter.getType(actual));
         writer.dup();
-        writer.invokeConstructor(constructor.owner.type, constructor.method);
+        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
 
         for (int index = 0; index < keys.size(); ++index) {
             AExpression key = keys.get(index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index a780ea3e05b..cf6f040c975 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -26,6 +26,7 @@ import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.PainlessMethodKey;
+import org.objectweb.asm.Type;
 
 import java.util.List;
 import java.util.Objects;
@@ -104,7 +105,7 @@ public final class ENewObj extends AExpression {
             argument.write(writer, globals);
         }
 
-        writer.invokeConstructor(constructor.owner.type, constructor.method);
+        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index 8eb154e745b..a1a0ee1dade 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -25,6 +25,7 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.objectweb.asm.Type;
 
 import java.lang.reflect.Modifier;
 import java.util.Objects;
@@ -63,9 +64,9 @@ final class PSubField extends AStoreable {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
-            writer.getStatic(field.owner.type, field.javaName, MethodWriter.getType(field.clazz));
+            writer.getStatic(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
         } else {
-            writer.getField(field.owner.type, field.javaName, MethodWriter.getType(field.clazz));
+            writer.getField(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
         }
     }
 
@@ -94,9 +95,9 @@ final class PSubField extends AStoreable {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
-            writer.getStatic(field.owner.type, field.javaName, MethodWriter.getType(field.clazz));
+            writer.getStatic(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
         } else {
-            writer.getField(field.owner.type, field.javaName, MethodWriter.getType(field.clazz));
+            writer.getField(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
         }
     }
 
@@ -105,9 +106,9 @@ final class PSubField extends AStoreable {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
-            writer.putStatic(field.owner.type, field.javaName, MethodWriter.getType(field.clazz));
+            writer.putStatic(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
         } else {
-            writer.putField(field.owner.type, field.javaName, MethodWriter.getType(field.clazz));
+            writer.putField(Type.getType(field.target), field.javaName, MethodWriter.getType(field.clazz));
         }
     }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index 5e8e6ad47d8..4486a52ccb1 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -27,6 +27,7 @@ import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.spi.Whitelist;
 
@@ -67,8 +68,8 @@ public class PainlessDocGenerator {
         Path indexPath = apiRootPath.resolve("index.asciidoc");
         logger.info("Starting to write [index.asciidoc]");
         try (PrintStream indexStream = new PrintStream(
-                Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
-                false, StandardCharsets.UTF_8.name())) {
+            Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
+            false, StandardCharsets.UTF_8.name())) {
             emitGeneratedWarning(indexStream);
             List<PainlessClass> structs = PAINLESS_LOOKUP.getStructs().stream().sorted(comparing(t -> t.name)).collect(toList());
             for (PainlessClass struct : structs) {
@@ -91,7 +92,7 @@ public class PainlessDocGenerator {
                         false, StandardCharsets.UTF_8.name())) {
                     emitGeneratedWarning(typeStream);
                     typeStream.print("[[");
-                    emitAnchor(typeStream, struct);
+                    emitAnchor(typeStream, struct.clazz);
                     typeStream.print("]]++");
                     typeStream.print(struct.name);
                     typeStream.println("++::");
@@ -104,10 +105,11 @@ public class PainlessDocGenerator {
                     struct.constructors.values().stream().sorted(NUMBER_OF_ARGS).forEach(documentMethod);
                     Map<String, PainlessClass> inherited = new TreeMap<>();
                     struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(method -> {
-                        if (method.owner == struct) {
+                        if (method.target == struct.clazz) {
                             documentMethod(typeStream, method);
                         } else {
-                            inherited.put(method.owner.name, method.owner);
+                            PainlessClass painlessClass = PAINLESS_LOOKUP.getPainlessStructFromJavaClass(method.target);
+                            inherited.put(painlessClass.name, painlessClass);
                         }
                     });
 
@@ -206,16 +208,16 @@ public class PainlessDocGenerator {
     /**
      * Anchor text for a {@link PainlessClass}.
      */
-    private static void emitAnchor(PrintStream stream, PainlessClass struct) {
+    private static void emitAnchor(PrintStream stream, Class<?> clazz) {
         stream.print("painless-api-reference-");
-        stream.print(struct.name.replace('.', '-'));
+        stream.print(PainlessLookupUtility.anyTypeToPainlessTypeName(clazz).replace('.', '-'));
     }
 
     /**
      * Anchor text for a {@link PainlessMethod}.
      */
     private static void emitAnchor(PrintStream stream, PainlessMethod method) {
-        emitAnchor(stream, method.owner);
+        emitAnchor(stream, method.target);
         stream.print('-');
         stream.print(methodName(method));
         stream.print('-');
@@ -226,18 +228,18 @@ public class PainlessDocGenerator {
      * Anchor text for a {@link PainlessField}.
      */
     private static void emitAnchor(PrintStream stream, PainlessField field) {
-        emitAnchor(stream, field.owner);
+        emitAnchor(stream, field.target);
         stream.print('-');
         stream.print(field.name);
     }
 
     private static String methodName(PainlessMethod method) {
-        return method.name.equals("<init>") ? method.owner.name : method.name;
+        return method.name.equals("<init>") ? PainlessLookupUtility.anyTypeToPainlessTypeName(method.target) : method.name;
     }
 
     /**
      * Emit a {@link Class}. If the type is primitive or an array of primitives this just emits the name of the type. Otherwise this emits
-       an internal link with the text.
+     an internal link with the text.
      */
     private static void emitType(PrintStream stream, Class<?> clazz) {
         emitStruct(stream, PAINLESS_LOOKUP.getPainlessStructFromJavaClass(clazz));
@@ -253,7 +255,7 @@ public class PainlessDocGenerator {
     private static void emitStruct(PrintStream stream, PainlessClass struct) {
         if (false == struct.clazz.isPrimitive() && false == struct.name.equals("def")) {
             stream.print("<<");
-            emitAnchor(stream, struct);
+            emitAnchor(stream, struct.clazz);
             stream.print(',');
             stream.print(struct.name);
             stream.print(">>");
@@ -271,14 +273,14 @@ public class PainlessDocGenerator {
         stream.print("link:{");
         stream.print(root);
         stream.print("-javadoc}/");
-        stream.print(classUrlPath(method.augmentation != null ? method.augmentation : method.owner.clazz));
+        stream.print(classUrlPath(method.augmentation != null ? method.augmentation : method.target));
         stream.print(".html#");
         stream.print(methodName(method));
         stream.print("%2D");
         boolean first = true;
         if (method.augmentation != null) {
             first = false;
-            stream.print(method.owner.clazz.getName());
+            stream.print(method.target.getName());
         }
         for (Class<?> clazz: method.arguments) {
             if (first) {
@@ -303,7 +305,7 @@ public class PainlessDocGenerator {
         stream.print("link:{");
         stream.print(root);
         stream.print("-javadoc}/");
-        stream.print(classUrlPath(field.owner.clazz));
+        stream.print(classUrlPath(field.target));
         stream.print(".html#");
         stream.print(field.javaName);
     }
@@ -315,21 +317,21 @@ public class PainlessDocGenerator {
         if (method.augmentation != null) {
             return "painless";
         }
-        return javadocRoot(method.owner);
+        return javadocRoot(method.target);
     }
 
     /**
      * Pick the javadoc root for a {@link PainlessField}.
      */
     private static String javadocRoot(PainlessField field) {
-        return javadocRoot(field.owner);
+        return javadocRoot(field.target);
     }
 
     /**
-     * Pick the javadoc root for a {@link PainlessClass}.
+     * Pick the javadoc root for a {@link Class<?>}.
      */
-    private static String javadocRoot(PainlessClass struct) {
-        String classPackage = struct.clazz.getPackage().getName();
+    private static String javadocRoot(Class<?> clazz) {
+        String classPackage = clazz.getPackage().getName();
         if (classPackage.startsWith("java")) {
             return "java8";
         }

From 1b97652a4cc1aa2acbe761464745abb35dd8faa0 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 17 Jul 2018 14:20:41 -0400
Subject: [PATCH 068/260] Build: Move shadow customizations into common code
 (#32014)

Moves the customizations to the build to produce nice shadow jars and
javadocs into common build code, mostly BuildPlugin with a little into
the root build.gradle file. This means that any project that applies the
shadow plugin will automatically be set up just like the high level rest
client:
* The non-shadow jar will not be built
* The shadow jar will not have a "classifier"
* Tests will run against the shadow jar
* Javadoc will include all of the shadowed classes
* Service files in `META-INF/services` will be merged
---
 benchmarks/build.gradle                       | 15 ---
 build.gradle                                  | 57 +++++++++--
 buildSrc/build.gradle                         |  1 +
 .../elasticsearch/gradle/BuildPlugin.groovy   | 92 ++++++++++++++++-
 .../gradle/plugin/PluginBuildPlugin.groovy    | 14 +++
 client/benchmark/build.gradle                 | 12 ---
 client/rest-high-level/build.gradle           | 99 -------------------
 x-pack/plugin/sql/jdbc/build.gradle           | 36 -------
 8 files changed, 153 insertions(+), 173 deletions(-)

diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index e7ee5a059ab..80d1982300d 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -17,17 +17,6 @@
  * under the License.
  */
 
-buildscript {
-    repositories {
-        maven {
-            url 'https://plugins.gradle.org/m2/'
-        }
-    }
-    dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
-    }
-}
-
 apply plugin: 'elasticsearch.build'
 
 // order of this section matters, see: https://github.com/johnrengelman/shadow/issues/336
@@ -81,10 +70,6 @@ thirdPartyAudit.excludes = [
         'org.openjdk.jmh.util.Utils'
 ]
 
-shadowJar {
-    classifier = 'benchmarks'
-}
-
 runShadow {
     executable = new File(project.runtimeJavaHome, 'bin/java')
 }
diff --git a/build.gradle b/build.gradle
index 187e2477052..ccbb6898dc4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-
+import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.BuildPlugin
@@ -303,18 +303,55 @@ subprojects {
     if (project.plugins.hasPlugin(BuildPlugin)) {
       String artifactsHost = VersionProperties.elasticsearch.isSnapshot() ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
       Closure sortClosure = { a, b -> b.group <=> a.group }
-      Closure depJavadocClosure = { dep ->
-        if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
-          Project upstreamProject = dependencyToProject(dep)
-          if (upstreamProject != null) {
-            project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
-            String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
-            project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${upstreamProject.buildDir}/docs/javadoc/"
+      Closure depJavadocClosure = { shadowed, dep ->
+        if (dep.group == null || false == dep.group.startsWith('org.elasticsearch')) {
+          return
+        }
+        Project upstreamProject = dependencyToProject(dep)
+        if (upstreamProject == null) {
+          return
+        }
+        if (shadowed) {
+          /*
+           * Include the source of shadowed upstream projects so we don't
+           * have to publish their javadoc.
+           */
+          project.evaluationDependsOn(upstreamProject.path)
+          project.javadoc.source += upstreamProject.javadoc.source
+          /*
+           * Do not add those projects to the javadoc classpath because
+           * we are going to resolve them with their source instead.
+           */
+          project.javadoc.classpath = project.javadoc.classpath.filter { f ->
+            false == upstreamProject.configurations.archives.artifacts.files.files.contains(f)
           }
+          /*
+           * Instead we need the upstream project's javadoc classpath so
+           * we don't barf on the classes that it references.
+           */
+          project.javadoc.classpath += upstreamProject.javadoc.classpath
+        } else {
+          // Link to non-shadowed dependant projects
+          project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
+          String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
+          project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${upstreamProject.buildDir}/docs/javadoc/"
         }
       }
-      project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
-      project.configurations.compileOnly.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+      boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin)
+      project.configurations.compile.dependencies
+          .findAll()
+          .toSorted(sortClosure)
+          .each({ c -> depJavadocClosure(hasShadow, c) })
+      project.configurations.compileOnly.dependencies
+          .findAll()
+          .toSorted(sortClosure)
+          .each({ c -> depJavadocClosure(hasShadow, c) })
+      if (hasShadow) {
+        project.configurations.shadow.dependencies
+            .findAll()
+            .toSorted(sortClosure)
+            .each({ c -> depJavadocClosure(false, c) })
+      }
     }
   }
 }
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 3d100daf7d6..eb95ff148f6 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -104,6 +104,7 @@ dependencies {
   compile 'de.thetaphi:forbiddenapis:2.5'
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
+  compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
   testCompile "junit:junit:${props.getProperty('junit')}"
 }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 89e10c50ff7..b5b5ec95bec 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
+import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.eclipse.jgit.lib.Constants
 import org.eclipse.jgit.lib.RepositoryBuilder
@@ -36,12 +37,14 @@ import org.gradle.api.artifacts.ModuleDependency
 import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
+import org.gradle.api.artifacts.SelfResolvingDependency
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.execution.TaskExecutionGraph
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
 import org.gradle.api.publish.maven.tasks.GenerateMavenPom
+import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.bundling.Jar
 import org.gradle.api.tasks.compile.GroovyCompile
 import org.gradle.api.tasks.compile.JavaCompile
@@ -498,7 +501,41 @@ class BuildPlugin implements Plugin<Project> {
                     }
                 }
             }
+            project.plugins.withType(ShadowPlugin).whenPluginAdded {
+                project.publishing {
+                    publications {
+                        nebula(MavenPublication) {
+                            artifact project.tasks.shadowJar
+                            artifactId = project.archivesBaseName
+                            /*
+                            * Configure the pom to include the "shadow" as compile dependencies
+                            * because that is how we're using them but remove all other dependencies
+                            * because they've been shaded into the jar.
+                            */
+                            pom.withXml { XmlProvider xml ->
+                                Node root = xml.asNode()
+                                root.remove(root.dependencies)
+                                Node dependenciesNode = root.appendNode('dependencies')
+                                project.configurations.shadow.allDependencies.each {
+                                    if (false == it instanceof SelfResolvingDependency) {
+                                        Node dependencyNode = dependenciesNode.appendNode('dependency')
+                                        dependencyNode.appendNode('groupId', it.group)
+                                        dependencyNode.appendNode('artifactId', it.name)
+                                        dependencyNode.appendNode('version', it.version)
+                                        dependencyNode.appendNode('scope', 'compile')
+                                    }
+                                }
+                                // Be tidy and remove the element if it is empty
+                                if (dependenciesNode.children.empty) {
+                                    root.remove(dependenciesNode)
+                                }
+                            }
+                        }
+                    }
+                }
+            }
         }
+
     }
 
     /** Adds compiler settings to the project */
@@ -660,6 +697,28 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
         }
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            /*
+             * When we use the shadow plugin we entirely replace the
+             * normal jar with the shadow jar so we no longer want to run
+             * the jar task.
+             */
+            project.tasks.jar.enabled = false
+            project.tasks.shadowJar {
+                /*
+                 * Replace the default "shadow" classifier with null
+                 * which will leave the classifier off of the file name.
+                 */
+                classifier = null
+                /*
+                 * Not all cases need service files merged but it is
+                 * better to be safe
+                 */
+                mergeServiceFiles()
+            }
+            // Make sure we assemble the shadow jar
+            project.tasks.assemble.dependsOn project.tasks.shadowJar
+        }
     }
 
     /** Returns a closure of common configuration shared by unit and integration tests. */
@@ -744,6 +803,18 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             exclude '**/*$*.class'
+
+            project.plugins.withType(ShadowPlugin).whenPluginAdded {
+                /*
+                 * If we make a shaded jar we test against it.
+                 */
+                classpath -= project.tasks.compileJava.outputs.files
+                classpath -= project.configurations.compile
+                classpath -= project.configurations.runtime
+                classpath += project.configurations.shadow
+                classpath += project.tasks.shadowJar.outputs.files
+                dependsOn project.tasks.shadowJar
+            }
         }
     }
 
@@ -766,7 +837,26 @@ class BuildPlugin implements Plugin<Project> {
             additionalTest.dependsOn(project.tasks.testClasses)
             test.dependsOn(additionalTest)
         });
-        return test
+
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            /*
+             * We need somewhere to configure dependencies that we don't wish
+             * to shade into the jar. The shadow plugin creates a "shadow"
+             * configuration which  is *almost* exactly that. It is never
+             * bundled into the shaded jar but is used for main source
+             * compilation. Unfortunately, by default it is not used for
+             * *test* source compilation and isn't used in tests at all. This
+             * change makes it available for test compilation.
+             *
+             * Note that this isn't going to work properly with qa projects
+             * but they have no business applying the shadow plugin in the
+             * firstplace.
+             */
+            SourceSet testSourceSet = project.sourceSets.findByName('test')
+            if (testSourceSet != null) {
+                testSourceSet.compileClasspath += project.configurations.shadow
+            }
+        }
     }
 
     private static configurePrecommit(Project project) {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index eb4da8d1f31..d76084bf22e 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -18,11 +18,13 @@
  */
 package org.elasticsearch.gradle.plugin
 
+import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import nebula.plugin.info.scm.ScmInfoPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
@@ -46,6 +48,18 @@ public class PluginBuildPlugin extends BuildPlugin {
     @Override
     public void apply(Project project) {
         super.apply(project)
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            /*
+             * We've not tested these plugins together and we're fairly sure
+             * they aren't going to work properly as is *and* we're not really
+             * sure *why* you'd want to shade stuff in plugins. So we throw an
+             * exception here to make you come and read this comment. If you
+             * have a need for shadow while building plugins then know that you
+             * are probably going to have to fight with gradle for a while....
+             */
+            throw new InvalidUserDataException('elasticsearch.esplugin is not '
+                    + 'compatible with com.github.johnrengelman.shadow');
+        }
         configureDependencies(project)
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle
index 77867f5e273..0c3238d9853 100644
--- a/client/benchmark/build.gradle
+++ b/client/benchmark/build.gradle
@@ -17,18 +17,6 @@
  * under the License.
  */
 
-buildscript {
-  repositories {
-    maven {
-      url 'https://plugins.gradle.org/m2/'
-    }
-  }
-  dependencies {
-    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
-  }
-}
-
-
 apply plugin: 'elasticsearch.build'
 // build an uberjar with all benchmarks
 apply plugin: 'com.github.johnrengelman.shadow'
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index 2fed806e98c..a1260894bf7 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -21,17 +21,6 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.gradle.api.internal.provider.Providers
 
-buildscript {
-  repositories {
-    maven {
-      url 'https://plugins.gradle.org/m2/'
-    }
-  }
-  dependencies {
-    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
-  }
-}
-
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'nebula.maven-base-publish'
@@ -45,49 +34,6 @@ archivesBaseName = 'elasticsearch-rest-high-level-client'
 Task copyRestSpec = RestIntegTestTask.createCopyRestSpecTask(project, Providers.FALSE)
 test.dependsOn(copyRestSpec)
 
-publishing {
-  publications {
-    nebula(MavenPublication) {
-      artifact shadowJar
-      artifactId = archivesBaseName
-      /*
-       * Configure the pom to include the "shadow" as compile dependencies
-       * because that is how we're using them but remove all other dependencies
-       * because they've been shaded into the jar.
-       */
-      pom.withXml { XmlProvider xml ->
-        Node root = xml.asNode()
-        root.remove(root.dependencies)
-        Node dependenciesNode = root.appendNode('dependencies')
-        project.configurations.shadow.allDependencies.each {
-          if (false == it instanceof SelfResolvingDependency) {
-            Node dependencyNode = dependenciesNode.appendNode('dependency')
-            dependencyNode.appendNode('groupId', it.group)
-            dependencyNode.appendNode('artifactId', it.name)
-            dependencyNode.appendNode('version', it.version)
-            dependencyNode.appendNode('scope', 'compile')
-          }
-        }
-      }
-    }
-  }
-}
-
-/*
- * We need somewhere to configure dependencies that we don't wish to shade
- * into the high level REST client. The shadow plugin creates a "shadow"
- * configuration which is *almost* exactly that. It is never bundled into
- * the shaded jar but is used for main source compilation. Unfortunately,
- * by default it is not used for *test* source compilation and isn't used
- * in tests at all. This change makes it available for test compilation.
- * A change below makes it available for testing.
- */
-sourceSets {
-  test {
-    compileClasspath += configurations.shadow
-  }
-}
-
 dependencies {
   /*
    * Everything in the "shadow" configuration is *not* copied into the
@@ -124,48 +70,3 @@ forbiddenApisMain {
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
   signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
 }
-
-shadowJar {
-  classifier = null
-  mergeServiceFiles()
-}
-
-// We don't need normal jar, we use shadow jar instead
-jar.enabled = false
-assemble.dependsOn shadowJar
-
-javadoc {
-  /*
-   * Bundle all of the javadoc from all of the shaded projects into this one
-   * so we don't *have* to publish javadoc for all of the "client" jars.
-   */
-  configurations.compile.dependencies.all { Dependency dep ->
-    Project p = dependencyToProject(dep)
-    if (p != null) {
-      evaluationDependsOn(p.path)
-      source += p.sourceSets.main.allJava
-    }
-  }
-}
-
-/*
- * Use the jar for testing so we have tests of the bundled jar.
- * Use the "shadow" configuration for testing because we need things
- * in it.
- */
-test {
-  classpath -= compileJava.outputs.files
-  classpath -= configurations.compile
-  classpath -= configurations.runtime
-  classpath += configurations.shadow
-  classpath += shadowJar.outputs.files
-  dependsOn shadowJar
-}
-integTestRunner {
-  classpath -= compileJava.outputs.files
-  classpath -= configurations.compile
-  classpath -= configurations.runtime
-  classpath += configurations.shadow
-  classpath += shadowJar.outputs.files
-  dependsOn shadowJar
-}
diff --git a/x-pack/plugin/sql/jdbc/build.gradle b/x-pack/plugin/sql/jdbc/build.gradle
index 9d27c2030d6..a0d9b24c507 100644
--- a/x-pack/plugin/sql/jdbc/build.gradle
+++ b/x-pack/plugin/sql/jdbc/build.gradle
@@ -1,15 +1,3 @@
-
-buildscript {
-    repositories {
-        maven {
-            url 'https://plugins.gradle.org/m2/'
-        }
-    }
-    dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
-    }
-}
-
 apply plugin: 'elasticsearch.build'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
@@ -49,7 +37,6 @@ dependencyLicenses {
 }
 
 shadowJar {
-    classifier = null
     relocate 'com.fasterxml', 'org.elasticsearch.fasterxml'
 }
 
@@ -70,26 +57,3 @@ artifacts  {
     nodeps nodepsJar
     archives shadowJar
 }
-
-publishing {
-    publications {
-        nebula(MavenPublication) {
-            artifact shadowJar
-            pom.withXml {
-                // Nebula is mistakenly including all dependencies that are already shadowed into the shadow jar
-                asNode().remove(asNode().dependencies)
-            }
-        }
-    }
-}
-
-assemble.dependsOn shadowJar
-
-// Use the jar for testing so the tests are more "real"
-test {
-    classpath -= compileJava.outputs.files
-    classpath -= configurations.compile
-    classpath -= configurations.runtime
-    classpath += shadowJar.outputs.files
-    dependsOn shadowJar
-}

From d170ab33150ff96ed14cc47ea61e44136cc2dc1f Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 17 Jul 2018 15:49:20 -0400
Subject: [PATCH 069/260] Disable C2 from using AVX-512 on JDK 10 (#32138)

The C2 compiler in JDK 10 appears to have an issue compiling to AVX-512
instructions (on hardware that supports such). As a workaround, this
commit adds a JVM flag on JDK 10+ to disable the use of AVX-512
instructions until a fix is introduced to the JDK. Instead, we use a
flag to enable AVX and AVX2 only.

Note: Based on my reading of the C2 code, this flag does not appear to
have any impact on hardware that does not support AVX2. I have tested
this manually on an Intel Atom C2538 processor that supports neither AVX
nor AVX2. I have also tested this manually on an Intel i5-3317U
processor that supports AVX but not AVX2.
---
 distribution/src/config/jvm.options | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
index c5c0f44caeb..e486735eb8f 100644
--- a/distribution/src/config/jvm.options
+++ b/distribution/src/config/jvm.options
@@ -100,3 +100,6 @@ ${error.file}
 # due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
 # time/date parsing will break in an incompatible way for some date patterns and locals
 9-:-Djava.locale.providers=COMPAT
+
+# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512
+10-:-XX:UseAVX=2

From 6371d51866e6b015f378fad568abbc0a7df94e2d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 17 Jul 2018 13:14:46 -0700
Subject: [PATCH 070/260] Build: Make additional test deps of check (#32015)

This commit moves additional unit test runners from being dependencies
of the test task to dependencies of check. Without this change,
reproduce lines are incorrect due to the additional test runner not
matching any of the reproduce class/method info.

closes #31964
---
 .../elasticsearch/gradle/BuildPlugin.groovy   |  2 +-
 server/build.gradle                           | 21 ++++++++++---------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index b5b5ec95bec..219d00ba640 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -835,7 +835,7 @@ class BuildPlugin implements Plugin<Project> {
             additionalTest.configure(commonTestConfig(project))
             additionalTest.configure(config)
             additionalTest.dependsOn(project.tasks.testClasses)
-            test.dependsOn(additionalTest)
+            project.check.dependsOn(additionalTest)
         });
 
         project.plugins.withType(ShadowPlugin).whenPluginAdded {
diff --git a/server/build.gradle b/server/build.gradle
index da60bca5a3e..7db073f43a5 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -156,6 +156,16 @@ if (isEclipse) {
 compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
+// TODO: remove ScriptDocValuesMissingV6BehaviourTests in 7.0
+additionalTest('testScriptDocValuesMissingV6Behaviour'){
+  include '**/ScriptDocValuesMissingV6BehaviourTests.class'
+  systemProperty 'es.scripting.exception_for_missing_value', 'false'
+}
+test {
+  // these are tested explicitly in separate test tasks
+  exclude '**/*ScriptDocValuesMissingV6BehaviourTests.class'
+}
+
 forbiddenPatterns {
   exclude '**/*.json'
   exclude '**/*.jmx'
@@ -329,7 +339,7 @@ if (isEclipse == false || project.path == ":server-tests") {
   task integTest(type: RandomizedTestingTask,
                  group: JavaBasePlugin.VERIFICATION_GROUP,
                  description: 'Multi-node tests',
-                 dependsOn: test.dependsOn.collect()) {
+                 dependsOn: test.dependsOn) {
     configure(BuildPlugin.commonTestConfig(project))
     classpath = project.test.classpath
     testClassesDirs = project.test.testClassesDirs
@@ -339,12 +349,3 @@ if (isEclipse == false || project.path == ":server-tests") {
   integTest.mustRunAfter test
 }
 
-// TODO: remove ScriptDocValuesMissingV6BehaviourTests in 7.0
-additionalTest('testScriptDocValuesMissingV6Behaviour'){
-  include '**/ScriptDocValuesMissingV6BehaviourTests.class'
-  systemProperty 'es.scripting.exception_for_missing_value', 'false'
-}
-test {
-  // these are tested explicitly in separate test tasks
-  exclude '**/*ScriptDocValuesMissingV6BehaviourTests.class'
-}

From 03c16cd0e33a3c869c2a0347147771b895ba20e8 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 17 Jul 2018 13:54:49 -0700
Subject: [PATCH 071/260] Painless: Add PainlessClassBuilder (#32141)

Several pieces of data in PainlessClass cannot be passed in at the time the
PainlessClass is created so it must be "frozen" after all the data is collected. This means
PainlessClass is currently serving two functions as both a builder and a set of data. This
separates the two pieces into clearly distinct values.

This change also removes the PainlessMethodKey in favor of a simple String. The goal is
to have the painless method key be completely internal to the PainlessLookup eventually
and this simplifies the way there. Note that this was added since PainlessClass and
PainlessClassBuilder were already being changed instead of a follow up PR.
---
 .../java/org/elasticsearch/painless/Def.java  |  3 +-
 .../elasticsearch/painless/FunctionRef.java   |  8 +-
 .../org/elasticsearch/painless/Locals.java    |  9 +--
 .../painless/lookup/PainlessClass.java        | 73 +++++------------
 .../painless/lookup/PainlessClassBuilder.java | 70 +++++++++++++++++
 .../lookup/PainlessLookupBuilder.java         | 78 ++++++++++---------
 .../painless/lookup/PainlessMethodKey.java    | 75 ------------------
 .../painless/node/ECallLocal.java             |  4 +-
 .../painless/node/EFunctionRef.java           |  4 +-
 .../painless/node/EListInit.java              |  9 ++-
 .../elasticsearch/painless/node/EMapInit.java |  9 ++-
 .../elasticsearch/painless/node/ENewObj.java  |  4 +-
 .../painless/node/PCallInvoke.java            |  3 +-
 .../elasticsearch/painless/node/PField.java   | 13 ++--
 .../painless/node/PSubListShortcut.java       |  6 +-
 .../painless/node/PSubMapShortcut.java        |  6 +-
 .../elasticsearch/painless/node/SSource.java  |  6 +-
 .../painless/node/SSubEachIterable.java       |  5 +-
 .../painless/node/NodeToStringTests.java      | 10 +--
 19 files changed, 178 insertions(+), 217 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethodKey.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index fe11ff4814b..f3388fc4bb2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -23,7 +23,6 @@ import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.lang.invoke.CallSite;
 import java.lang.invoke.MethodHandle;
@@ -185,7 +184,7 @@ public final class Def {
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
     static PainlessMethod lookupMethodInternal(PainlessLookup painlessLookup, Class<?> receiverClass, String name, int arity) {
-        PainlessMethodKey key = new PainlessMethodKey(name, arity);
+        String key = PainlessLookupUtility.buildPainlessMethodKey(name, arity);
         // check whitelist for matching method
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
             PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 925359fabc5..d64e833912f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -23,7 +23,6 @@ import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.objectweb.asm.Type;
 
 import java.lang.invoke.MethodType;
@@ -177,10 +176,11 @@ public class FunctionRef {
         final PainlessMethod impl;
         // ctor ref
         if ("new".equals(call)) {
-            impl = struct.constructors.get(new PainlessMethodKey("<init>", method.arguments.size()));
+            impl = struct.constructors.get(PainlessLookupUtility.buildPainlessMethodKey("<init>", method.arguments.size()));
         } else {
             // look for a static impl first
-            PainlessMethod staticImpl = struct.staticMethods.get(new PainlessMethodKey(call, method.arguments.size()));
+            PainlessMethod staticImpl =
+                    struct.staticMethods.get(PainlessLookupUtility.buildPainlessMethodKey(call, method.arguments.size()));
             if (staticImpl == null) {
                 // otherwise a virtual impl
                 final int arity;
@@ -191,7 +191,7 @@ public class FunctionRef {
                     // receiver passed
                     arity = method.arguments.size() - 1;
                 }
-                impl = struct.methods.get(new PainlessMethodKey(call, arity));
+                impl = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(call, arity));
             } else {
                 impl = staticImpl;
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index e797740fed1..6c1010a3450 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -23,7 +23,6 @@ import org.elasticsearch.painless.ScriptClassInfo.MethodArgument;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -144,7 +143,7 @@ public final class Locals {
     }
 
     /** Looks up a method. Returns null if the method does not exist. */
-    public PainlessMethod getMethod(PainlessMethodKey key) {
+    public PainlessMethod getMethod(String key) {
         PainlessMethod method = lookupMethod(key);
         if (method != null) {
             return method;
@@ -200,7 +199,7 @@ public final class Locals {
     // variable name -> variable
     private Map<String,Variable> variables;
     // method name+arity -> methods
-    private Map<PainlessMethodKey,PainlessMethod> methods;
+    private Map<String,PainlessMethod> methods;
 
     /**
      * Create a new Locals
@@ -238,7 +237,7 @@ public final class Locals {
     }
 
     /** Looks up a method at this scope only. Returns null if the method does not exist. */
-    private PainlessMethod lookupMethod(PainlessMethodKey key) {
+    private PainlessMethod lookupMethod(String key) {
         if (methods == null) {
             return null;
         }
@@ -261,7 +260,7 @@ public final class Locals {
         if (methods == null) {
             methods = new HashMap<>();
         }
-        methods.put(new PainlessMethodKey(method.name, method.arguments.size()), method);
+        methods.put(PainlessLookupUtility.buildPainlessMethodKey(method.name, method.arguments.size()), method);
         // TODO: check result
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
index 7d84899b00e..57b18bc60da 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
@@ -19,19 +19,20 @@
 
 package org.elasticsearch.painless.lookup;
 
+import org.objectweb.asm.Type;
+
 import java.lang.invoke.MethodHandle;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 
 public final class PainlessClass {
     public final String name;
     public final Class<?> clazz;
-    public final org.objectweb.asm.Type type;
+    public final Type type;
 
-    public final Map<PainlessMethodKey, PainlessMethod> constructors;
-    public final Map<PainlessMethodKey, PainlessMethod> staticMethods;
-    public final Map<PainlessMethodKey, PainlessMethod> methods;
+    public final Map<String, PainlessMethod> constructors;
+    public final Map<String, PainlessMethod> staticMethods;
+    public final Map<String, PainlessMethod> methods;
 
     public final Map<String, PainlessField> staticMembers;
     public final Map<String, PainlessField> members;
@@ -41,63 +42,25 @@ public final class PainlessClass {
 
     public final PainlessMethod functionalMethod;
 
-    PainlessClass(String name, Class<?> clazz, org.objectweb.asm.Type type) {
+    PainlessClass(String name, Class<?> clazz, Type type,
+                  Map<String, PainlessMethod> constructors, Map<String, PainlessMethod> staticMethods, Map<String, PainlessMethod> methods,
+                  Map<String, PainlessField> staticMembers, Map<String, PainlessField> members,
+                  Map<String, MethodHandle> getters, Map<String, MethodHandle> setters,
+                  PainlessMethod functionalMethod) {
         this.name = name;
         this.clazz = clazz;
         this.type = type;
 
-        constructors = new HashMap<>();
-        staticMethods = new HashMap<>();
-        methods = new HashMap<>();
+        this.constructors = Collections.unmodifiableMap(constructors);
+        this.staticMethods = Collections.unmodifiableMap(staticMethods);
+        this.methods = Collections.unmodifiableMap(methods);
 
-        staticMembers = new HashMap<>();
-        members = new HashMap<>();
+        this.staticMembers = Collections.unmodifiableMap(staticMembers);
+        this.members = Collections.unmodifiableMap(members);
 
-        getters = new HashMap<>();
-        setters = new HashMap<>();
-
-        functionalMethod = null;
-    }
-
-    private PainlessClass(PainlessClass struct, PainlessMethod functionalMethod) {
-        name = struct.name;
-        clazz = struct.clazz;
-        type = struct.type;
-
-        constructors = Collections.unmodifiableMap(struct.constructors);
-        staticMethods = Collections.unmodifiableMap(struct.staticMethods);
-        methods = Collections.unmodifiableMap(struct.methods);
-
-        staticMembers = Collections.unmodifiableMap(struct.staticMembers);
-        members = Collections.unmodifiableMap(struct.members);
-
-        getters = Collections.unmodifiableMap(struct.getters);
-        setters = Collections.unmodifiableMap(struct.setters);
+        this.getters = Collections.unmodifiableMap(getters);
+        this.setters = Collections.unmodifiableMap(setters);
 
         this.functionalMethod = functionalMethod;
     }
-
-    public PainlessClass freeze(PainlessMethod functionalMethod) {
-        return new PainlessClass(this, functionalMethod);
-    }
-
-    @Override
-    public boolean equals(Object object) {
-        if (this == object) {
-            return true;
-        }
-
-        if (object == null || getClass() != object.getClass()) {
-            return false;
-        }
-
-        PainlessClass struct = (PainlessClass)object;
-
-        return name.equals(struct.name);
-    }
-
-    @Override
-    public int hashCode() {
-        return name.hashCode();
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
new file mode 100644
index 00000000000..0eda3660f0b
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.lookup;
+
+import org.objectweb.asm.Type;
+
+import java.lang.invoke.MethodHandle;
+import java.util.HashMap;
+import java.util.Map;
+
+final class PainlessClassBuilder {
+    final String name;
+    final Class<?> clazz;
+    final Type type;
+
+    final Map<String, PainlessMethod> constructors;
+    final Map<String, PainlessMethod> staticMethods;
+    final Map<String, PainlessMethod> methods;
+
+    final Map<String, PainlessField> staticMembers;
+    final Map<String, PainlessField> members;
+
+    final Map<String, MethodHandle> getters;
+    final Map<String, MethodHandle> setters;
+
+    PainlessMethod functionalMethod;
+
+    PainlessClassBuilder(String name, Class<?> clazz, Type type) {
+        this.name = name;
+        this.clazz = clazz;
+        this.type = type;
+
+        constructors = new HashMap<>();
+        staticMethods = new HashMap<>();
+        methods = new HashMap<>();
+
+        staticMembers = new HashMap<>();
+        members = new HashMap<>();
+
+        getters = new HashMap<>();
+        setters = new HashMap<>();
+
+        functionalMethod = null;
+    }
+
+    PainlessClass build() {
+        return new PainlessClass(name, clazz, type,
+            constructors, staticMethods, methods,
+            staticMembers, members,
+            getters, setters,
+            functionalMethod);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 5641eee1b5d..2150c0b210a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -37,6 +37,8 @@ import java.util.Map;
 import java.util.Stack;
 import java.util.regex.Pattern;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
+
 public class PainlessLookupBuilder {
     private static final Pattern TYPE_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$");
 
@@ -60,16 +62,16 @@ public class PainlessLookupBuilder {
     }
 
     private final Map<String, Class<?>> painlessTypesToJavaClasses;
-    private final Map<Class<?>, PainlessClass> javaClassesToPainlessStructs;
+    private final Map<Class<?>, PainlessClassBuilder> javaClassesToPainlessClassBuilders;
 
     public PainlessLookupBuilder(List<Whitelist> whitelists) {
         painlessTypesToJavaClasses = new HashMap<>();
-        javaClassesToPainlessStructs = new HashMap<>();
+        javaClassesToPainlessClassBuilders = new HashMap<>();
 
         String origin = null;
 
         painlessTypesToJavaClasses.put("def", def.class);
-        javaClassesToPainlessStructs.put(def.class, new PainlessClass("def", Object.class, Type.getType(Object.class)));
+        javaClassesToPainlessClassBuilders.put(def.class, new PainlessClassBuilder("def", Object.class, Type.getType(Object.class)));
 
         try {
             // first iteration collects all the Painless type names that
@@ -77,7 +79,8 @@ public class PainlessLookupBuilder {
             for (Whitelist whitelist : whitelists) {
                 for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
                     String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-                    PainlessClass painlessStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(painlessTypeName));
+                    PainlessClassBuilder painlessStruct =
+                            javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(painlessTypeName));
 
                     if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
                         throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
@@ -87,8 +90,8 @@ public class PainlessLookupBuilder {
                     origin = whitelistStruct.origin;
                     addStruct(whitelist.javaClassLoader, whitelistStruct);
 
-                    painlessStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(painlessTypeName));
-                    javaClassesToPainlessStructs.put(painlessStruct.clazz, painlessStruct);
+                    painlessStruct = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(painlessTypeName));
+                    javaClassesToPainlessClassBuilders.put(painlessStruct.clazz, painlessStruct);
                 }
             }
 
@@ -121,8 +124,8 @@ public class PainlessLookupBuilder {
 
         // goes through each Painless struct and determines the inheritance list,
         // and then adds all inherited types to the Painless struct's whitelist
-        for (Class<?> javaClass : javaClassesToPainlessStructs.keySet()) {
-            PainlessClass painlessStruct = javaClassesToPainlessStructs.get(javaClass);
+        for (Class<?> javaClass : javaClassesToPainlessClassBuilders.keySet()) {
+            PainlessClassBuilder painlessStruct = javaClassesToPainlessClassBuilders.get(javaClass);
 
             List<String> painlessSuperStructs = new ArrayList<>();
             Class<?> javaSuperClass = painlessStruct.clazz.getSuperclass();
@@ -133,7 +136,7 @@ public class PainlessLookupBuilder {
             // adds super classes to the inheritance list
             if (javaSuperClass != null && javaSuperClass.isInterface() == false) {
                 while (javaSuperClass != null) {
-                    PainlessClass painlessSuperStruct = javaClassesToPainlessStructs.get(javaSuperClass);
+                    PainlessClassBuilder painlessSuperStruct = javaClassesToPainlessClassBuilders.get(javaSuperClass);
 
                     if (painlessSuperStruct != null) {
                         painlessSuperStructs.add(painlessSuperStruct.name);
@@ -149,7 +152,7 @@ public class PainlessLookupBuilder {
                 Class<?> javaInterfaceLookup = javaInteraceLookups.pop();
 
                 for (Class<?> javaSuperInterface : javaInterfaceLookup.getInterfaces()) {
-                    PainlessClass painlessInterfaceStruct = javaClassesToPainlessStructs.get(javaSuperInterface);
+                    PainlessClassBuilder painlessInterfaceStruct = javaClassesToPainlessClassBuilders.get(javaSuperInterface);
 
                     if (painlessInterfaceStruct != null) {
                         String painlessInterfaceStructName = painlessInterfaceStruct.name;
@@ -170,7 +173,7 @@ public class PainlessLookupBuilder {
 
             // copies methods and fields from Object into interface types
             if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) {
-                PainlessClass painlessObjectStruct = javaClassesToPainlessStructs.get(Object.class);
+                PainlessClassBuilder painlessObjectStruct = javaClassesToPainlessClassBuilders.get(Object.class);
 
                 if (painlessObjectStruct != null) {
                     copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name));
@@ -179,14 +182,9 @@ public class PainlessLookupBuilder {
         }
 
         // precompute runtime classes
-        for (PainlessClass painlessStruct : javaClassesToPainlessStructs.values()) {
+        for (PainlessClassBuilder painlessStruct : javaClassesToPainlessClassBuilders.values()) {
             addRuntimeClass(painlessStruct);
         }
-
-        // copy all structs to make them unmodifiable for outside users:
-        for (Map.Entry<Class<?>,PainlessClass> entry : javaClassesToPainlessStructs.entrySet()) {
-            entry.setValue(entry.getValue().freeze(computeFunctionalInterfaceMethod(entry.getValue())));
-        }
     }
 
     private void addStruct(ClassLoader whitelistClassLoader, WhitelistClass whitelistStruct) {
@@ -223,12 +221,12 @@ public class PainlessLookupBuilder {
             }
         }
 
-        PainlessClass existingStruct = javaClassesToPainlessStructs.get(javaClass);
+        PainlessClassBuilder existingStruct = javaClassesToPainlessClassBuilders.get(javaClass);
 
         if (existingStruct == null) {
-            PainlessClass struct = new PainlessClass(painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass));
+            PainlessClassBuilder struct = new PainlessClassBuilder(painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass));
             painlessTypesToJavaClasses.put(painlessTypeName, javaClass);
-            javaClassesToPainlessStructs.put(javaClass, struct);
+            javaClassesToPainlessClassBuilders.put(javaClass, struct);
         } else if (existingStruct.clazz.equals(javaClass) == false) {
             throw new IllegalArgumentException("struct [" + painlessTypeName + "] is used to " +
                 "illegally represent multiple java classes [" + whitelistStruct.javaClassName + "] and " +
@@ -261,7 +259,7 @@ public class PainlessLookupBuilder {
     }
 
     private void addConstructor(String ownerStructName, WhitelistConstructor whitelistConstructor) {
-        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for constructor with " +
@@ -295,7 +293,7 @@ public class PainlessLookupBuilder {
                 " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames, exception);
         }
 
-        PainlessMethodKey painlessMethodKey = new PainlessMethodKey("<init>", whitelistConstructor.painlessParameterTypeNames.size());
+        String painlessMethodKey = buildPainlessMethodKey("<init>", whitelistConstructor.painlessParameterTypeNames.size());
         PainlessMethod painlessConstructor = ownerStruct.constructors.get(painlessMethodKey);
 
         if (painlessConstructor == null) {
@@ -321,7 +319,7 @@ public class PainlessLookupBuilder {
     }
 
     private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, WhitelistMethod whitelistMethod) {
-        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
@@ -400,8 +398,8 @@ public class PainlessLookupBuilder {
                 "and parameters " + whitelistMethod.painlessParameterTypeNames);
         }
 
-        PainlessMethodKey painlessMethodKey =
-            new PainlessMethodKey(whitelistMethod.javaMethodName, whitelistMethod.painlessParameterTypeNames.size());
+        String painlessMethodKey =
+            buildPainlessMethodKey(whitelistMethod.javaMethodName, whitelistMethod.painlessParameterTypeNames.size());
 
         if (javaAugmentedClass == null && Modifier.isStatic(javaMethod.getModifiers())) {
             PainlessMethod painlessMethod = ownerStruct.staticMethods.get(painlessMethodKey);
@@ -459,7 +457,7 @@ public class PainlessLookupBuilder {
     }
 
     private void addField(String ownerStructName, WhitelistField whitelistField) {
-        PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
@@ -540,14 +538,14 @@ public class PainlessLookupBuilder {
     }
 
     private void copyStruct(String struct, List<String> children) {
-        final PainlessClass owner = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(struct));
+        final PainlessClassBuilder owner = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(struct));
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
         }
 
         for (int count = 0; count < children.size(); ++count) {
-            final PainlessClass child = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(children.get(count)));
+            final PainlessClassBuilder child = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(children.get(count)));
 
             if (child == null) {
                 throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
@@ -559,8 +557,8 @@ public class PainlessLookupBuilder {
                     " is not a super type of owner struct [" + owner.name + "] in copy.");
             }
 
-            for (Map.Entry<PainlessMethodKey,PainlessMethod> kvPair : child.methods.entrySet()) {
-                PainlessMethodKey methodKey = kvPair.getKey();
+            for (Map.Entry<String,PainlessMethod> kvPair : child.methods.entrySet()) {
+                String methodKey = kvPair.getKey();
                 PainlessMethod method = kvPair.getValue();
                 if (owner.methods.get(methodKey) == null) {
                     // TODO: some of these are no longer valid or outright don't work
@@ -625,10 +623,10 @@ public class PainlessLookupBuilder {
     /**
      * Precomputes a more efficient structure for dynamic method/field access.
      */
-    private void addRuntimeClass(final PainlessClass struct) {
+    private void addRuntimeClass(final PainlessClassBuilder struct) {
         // add all getters/setters
-        for (Map.Entry<PainlessMethodKey, PainlessMethod> method : struct.methods.entrySet()) {
-            String name = method.getKey().name;
+        for (Map.Entry<String, PainlessMethod> method : struct.methods.entrySet()) {
+            String name = method.getValue().name;
             PainlessMethod m = method.getValue();
 
             if (m.arguments.size() == 0 &&
@@ -668,7 +666,7 @@ public class PainlessLookupBuilder {
     }
 
     /** computes the functional interface method for a class, or returns null */
-    private PainlessMethod computeFunctionalInterfaceMethod(PainlessClass clazz) {
+    private PainlessMethod computeFunctionalInterfaceMethod(PainlessClassBuilder clazz) {
         if (!clazz.clazz.isInterface()) {
             return null;
         }
@@ -703,7 +701,7 @@ public class PainlessLookupBuilder {
         }
         // inspect the one method found from the reflection API, it should match the whitelist!
         java.lang.reflect.Method oneMethod = methods.get(0);
-        PainlessMethod painless = clazz.methods.get(new PainlessMethodKey(oneMethod.getName(), oneMethod.getParameterCount()));
+        PainlessMethod painless = clazz.methods.get(buildPainlessMethodKey(oneMethod.getName(), oneMethod.getParameterCount()));
         if (painless == null || painless.method.equals(org.objectweb.asm.commons.Method.getMethod(oneMethod)) == false) {
             throw new IllegalArgumentException("Class: " + clazz.name + " is functional but the functional " +
                 "method is not whitelisted!");
@@ -712,7 +710,15 @@ public class PainlessLookupBuilder {
     }
 
     public PainlessLookup build() {
-        return new PainlessLookup(painlessTypesToJavaClasses, javaClassesToPainlessStructs);
+        Map<Class<?>, PainlessClass> javaClassesToPainlessClasses = new HashMap<>();
+
+        // copy all structs to make them unmodifiable for outside users:
+        for (Map.Entry<Class<?>,PainlessClassBuilder> entry : javaClassesToPainlessClassBuilders.entrySet()) {
+            entry.getValue().functionalMethod = computeFunctionalInterfaceMethod(entry.getValue());
+            javaClassesToPainlessClasses.put(entry.getKey(), entry.getValue().build());
+        }
+
+        return new PainlessLookup(painlessTypesToJavaClasses, javaClassesToPainlessClasses);
     }
 
     public Class<?> getJavaClassFromPainlessType(String painlessType) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethodKey.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethodKey.java
deleted file mode 100644
index 49413ab0c5f..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethodKey.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless.lookup;
-
-import java.util.Objects;
-
-/**
- * Key for looking up a method.
- * <p>
- * Methods are keyed on both name and arity, and can be overloaded once per arity.
- * This allows signatures such as {@code String.indexOf(String) vs String.indexOf(String, int)}.
- * <p>
- * It is less flexible than full signature overloading where types can differ too, but
- * better than just the name, and overloading types adds complexity to users, too.
- */
-public final class PainlessMethodKey {
-    public final String name;
-    public final int arity;
-
-    /**
-     * Create a new lookup key
-     * @param name name of the method
-     * @param arity number of parameters
-     */
-    public PainlessMethodKey(String name, int arity) {
-        this.name = Objects.requireNonNull(name);
-        this.arity = arity;
-    }
-
-    @Override
-    public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + arity;
-        result = prime * result + name.hashCode();
-        return result;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) return true;
-        if (obj == null) return false;
-        if (getClass() != obj.getClass()) return false;
-        PainlessMethodKey other = (PainlessMethodKey) obj;
-        if (arity != other.arity) return false;
-        if (!name.equals(other.name)) return false;
-        return true;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append(name);
-        sb.append('/');
-        sb.append(arity);
-        return sb.toString();
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index dfed0ca47b4..098c75386e1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -23,8 +23,8 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.util.List;
 import java.util.Objects;
@@ -58,7 +58,7 @@ public final class ECallLocal extends AExpression {
 
     @Override
     void analyze(Locals locals) {
-        PainlessMethodKey methodKey = new PainlessMethodKey(name, arguments.size());
+        String methodKey = PainlessLookupUtility.buildPainlessMethodKey(name, arguments.size());
         method = locals.getMethod(methodKey);
 
         if (method == null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index d4eddb059a8..92b14a885a1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -27,7 +27,6 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.objectweb.asm.Type;
 
 import java.util.Objects;
@@ -71,7 +70,8 @@ public final class EFunctionRef extends AExpression implements ILambda {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                 "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], not a functional interface");
                     }
-                    PainlessMethod delegateMethod = locals.getMethod(new PainlessMethodKey(call, interfaceMethod.arguments.size()));
+                    PainlessMethod delegateMethod =
+                            locals.getMethod(PainlessLookupUtility.buildPainlessMethodKey(call, interfaceMethod.arguments.size()));
                     if (delegateMethod == null) {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                 "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], function not found");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index 7e923e5f90f..e0af653d209 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -23,8 +23,8 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Type;
 
@@ -62,14 +62,15 @@ public final class EListInit extends AExpression {
 
         actual = ArrayList.class;
 
-        constructor =
-                locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors.get(new PainlessMethodKey("<init>", 0));
+        constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors
+                .get(PainlessLookupUtility.buildPainlessMethodKey("<init>", 0));
 
         if (constructor == null) {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods.get(new PainlessMethodKey("add", 1));
+        method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods
+                .get(PainlessLookupUtility.buildPainlessMethodKey("add", 1));
 
         if (method == null) {
             throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index b350a758944..d81f08dc3cc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -23,8 +23,8 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Type;
 
@@ -68,14 +68,15 @@ public final class EMapInit extends AExpression {
 
         actual = HashMap.class;
 
-        constructor =
-                locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors.get(new PainlessMethodKey("<init>", 0));
+        constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors
+                .get(PainlessLookupUtility.buildPainlessMethodKey("<init>", 0));
 
         if (constructor == null) {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods.get(new PainlessMethodKey("put", 2));
+        method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods
+                .get(PainlessLookupUtility.buildPainlessMethodKey("put", 2));
 
         if (method == null) {
             throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index cf6f040c975..c0d4433f7fb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -24,8 +24,8 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.objectweb.asm.Type;
 
 import java.util.List;
@@ -65,7 +65,7 @@ public final class ENewObj extends AExpression {
         }
 
         PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual);
-        constructor = struct.constructors.get(new PainlessMethodKey("<init>", arguments.size()));
+        constructor = struct.constructors.get(PainlessLookupUtility.buildPainlessMethodKey("<init>", arguments.size()));
 
         if (constructor != null) {
             Class<?>[] types = new Class<?>[constructor.arguments.size()];
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index 445c053347e..cd5d6483791 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -26,7 +26,6 @@ import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.def;
 
 import java.util.List;
@@ -77,7 +76,7 @@ public final class PCallInvoke extends AExpression {
             struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(PainlessLookupUtility.getBoxedAnyType(prefix.actual));
         }
 
-        PainlessMethodKey methodKey = new PainlessMethodKey(name, arguments.size());
+        String methodKey = PainlessLookupUtility.buildPainlessMethodKey(name, arguments.size());
         PainlessMethod method = prefix instanceof EStatic ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey);
 
         if (method != null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index 3f2f8879564..b5df74358d3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -27,7 +27,6 @@ import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.def;
 
 import java.util.List;
@@ -74,16 +73,16 @@ public final class PField extends AStoreable {
             if (field != null) {
                 sub = new PSubField(location, field);
             } else {
-                PainlessMethod getter = struct.methods.get(
-                    new PainlessMethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
+                PainlessMethod getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(
+                        "get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
 
                 if (getter == null) {
-                    getter = struct.methods.get(
-                        new PainlessMethodKey("is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
+                    getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(
+                            "is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
                 }
 
-                PainlessMethod setter = struct.methods.get(
-                    new PainlessMethodKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
+                PainlessMethod setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(
+                        "set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
                 if (getter != null || setter != null) {
                     sub = new PSubShortcut(location, value, PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual), getter, setter);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index 0a3ab142ddc..3841b1fece1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -25,8 +25,8 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.util.Objects;
 import java.util.Set;
@@ -56,8 +56,8 @@ final class PSubListShortcut extends AStoreable {
 
     @Override
     void analyze(Locals locals) {
-        getter = struct.methods.get(new PainlessMethodKey("get", 1));
-        setter = struct.methods.get(new PainlessMethodKey("set", 2));
+        getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1));
+        setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("set", 2));
 
         if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1 ||
             getter.arguments.get(0) != int.class)) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index f71e2ac5d1f..13a3b9c9b94 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -24,8 +24,8 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 
 import java.util.Objects;
 import java.util.Set;
@@ -55,8 +55,8 @@ final class PSubMapShortcut extends AStoreable {
 
     @Override
     void analyze(Locals locals) {
-        getter = struct.methods.get(new PainlessMethodKey("get", 1));
-        setter = struct.methods.get(new PainlessMethodKey("put", 2));
+        getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1));
+        setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("put", 2));
 
         if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1)) {
             throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + struct.name + "]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index cd473e2c84e..c354e78a961 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -30,8 +30,8 @@ import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.SimpleChecksAdapter;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.node.SFunction.FunctionReserved;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
@@ -165,12 +165,12 @@ public final class SSource extends AStatement {
     }
 
     public void analyze(PainlessLookup painlessLookup) {
-        Map<PainlessMethodKey, PainlessMethod> methods = new HashMap<>();
+        Map<String, PainlessMethod> methods = new HashMap<>();
 
         for (SFunction function : functions) {
             function.generateSignature(painlessLookup);
 
-            PainlessMethodKey key = new PainlessMethodKey(function.name, function.parameters.size());
+            String key = PainlessLookupUtility.buildPainlessMethodKey(function.name, function.parameters.size());
 
             if (methods.put(key, function.method) != null) {
                 throw createError(new IllegalArgumentException("Duplicate functions with name [" + function.name + "]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index cec1297a4c4..798b30e2b6d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -29,7 +29,6 @@ import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
@@ -77,8 +76,8 @@ final class SSubEachIterable extends AStatement {
         if (expression.actual == def.class) {
             method = null;
         } else {
-            method = locals.getPainlessLookup().
-                    getPainlessStructFromJavaClass(expression.actual).methods.get(new PainlessMethodKey("iterator", 0));
+            method = locals.getPainlessLookup().getPainlessStructFromJavaClass(expression.actual).methods
+                    .get(PainlessLookupUtility.buildPainlessMethodKey("iterator", 0));
 
             if (method == null) {
                 throw createError(new IllegalArgumentException("Unable to create iterator for the type " +
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index 86d365e0fcc..cd3e4123e12 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -24,8 +24,8 @@ import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessCast;
 import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessMethodKey;
 import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.FeatureTest;
 import org.elasticsearch.painless.GenericElasticsearchScript;
@@ -405,14 +405,14 @@ public class NodeToStringTests extends ESTestCase {
     public void testPSubCallInvoke() {
         Location l = new Location(getTestName(), 0);
         PainlessClass c = painlessLookup.getPainlessStructFromJavaClass(Integer.class);
-        PainlessMethod m = c.methods.get(new PainlessMethodKey("toString", 0));
+        PainlessMethod m = c.methods.get(PainlessLookupUtility.buildPainlessMethodKey("toString", 0));
         PSubCallInvoke node = new PSubCallInvoke(l, m, null, emptyList());
         node.prefix = new EVariable(l, "a");
         assertEquals("(PSubCallInvoke (EVariable a) toString)", node.toString());
         assertEquals("(PSubNullSafeCallInvoke (PSubCallInvoke (EVariable a) toString))", new PSubNullSafeCallInvoke(l, node).toString());
 
         l = new Location(getTestName(), 1);
-        m = c.methods.get(new PainlessMethodKey("equals", 1));
+        m = c.methods.get(PainlessLookupUtility.buildPainlessMethodKey("equals", 1));
         node = new PSubCallInvoke(l, m, null, singletonList(new EVariable(l, "b")));
         node.prefix = new EVariable(l, "a");
         assertEquals("(PSubCallInvoke (EVariable a) equals (Args (EVariable b)))", node.toString());
@@ -502,8 +502,8 @@ public class NodeToStringTests extends ESTestCase {
     public void testPSubShortcut() {
         Location l = new Location(getTestName(), 0);
         PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(FeatureTest.class);
-        PainlessMethod getter = s.methods.get(new PainlessMethodKey("getX", 0));
-        PainlessMethod setter = s.methods.get(new PainlessMethodKey("setX", 1));
+        PainlessMethod getter = s.methods.get(PainlessLookupUtility.buildPainlessMethodKey("getX", 0));
+        PainlessMethod setter = s.methods.get(PainlessLookupUtility.buildPainlessMethodKey("setX", 1));
         PSubShortcut node = new PSubShortcut(l, "x", FeatureTest.class.getName(), getter, setter);
         node.prefix = new EVariable(l, "a");
         assertEquals("(PSubShortcut (EVariable a) x)", node.toString());

From 91d8371325bfd4a6cbf88d4e77f4769f1d8bdea0 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 17 Jul 2018 18:41:31 -0400
Subject: [PATCH 072/260] Build: Skip jar tests if jar disabled

The shadow plugin disables the jar task but we still attempted to extract
the jar to see if it had the right license and notice file. This skips
the extraction and those tests if the jar is built for any reason which
fixes projects that use the shadow plugin.
---
 build.gradle | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/build.gradle b/build.gradle
index ccbb6898dc4..90a9d88ac8b 100644
--- a/build.gradle
+++ b/build.gradle
@@ -574,6 +574,7 @@ subprojects { project ->
       commandLine "${->new File(rootProject.compilerJavaHome, 'bin/jar')}",
           'xf', "${-> jarTask.outputs.files.singleFile}", 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'
       workingDir destination
+      onlyIf {jarTask.enabled}
       doFirst {
         project.delete(destination)
         Files.createDirectories(destination)
@@ -582,6 +583,7 @@ subprojects { project ->
 
     final Task checkNotice = project.task("verify${jarTask.name.capitalize()}Notice") {
       dependsOn extract
+      onlyIf {jarTask.enabled}
       doLast {
         final List<String> noticeLines = Files.readAllLines(project.noticeFile.toPath())
         final Path noticePath = extract.destination.resolve('META-INF/NOTICE.txt')
@@ -592,6 +594,7 @@ subprojects { project ->
 
     final Task checkLicense = project.task("verify${jarTask.name.capitalize()}License") {
       dependsOn extract
+      onlyIf {jarTask.enabled}
       doLast {
         final List<String> licenseLines = Files.readAllLines(project.licenseFile.toPath())
         final Path licensePath = extract.destination.resolve('META-INF/LICENSE.txt')

From 351bbb89063dc00dbea76f8e09300ede9d1be28c Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 17 Jul 2018 20:25:27 -0400
Subject: [PATCH 073/260] Switch distribution to new style Requests (#30595)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `distribution/archives/integ-test-zip` project
to use the new versions.
---
 .../test/rest/CreatedLocationHeaderIT.java    | 24 +++++-----
 .../test/rest/NodeRestUsageIT.java            | 44 +++++++++----------
 .../test/rest/RequestsWithoutContentIT.java   | 37 ++++++++--------
 .../rest/WaitForRefreshAndCloseTests.java     | 42 +++++++++---------
 4 files changed, 74 insertions(+), 73 deletions(-)

diff --git a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
index 9c9b6af705a..71a41db80a2 100644
--- a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
+++ b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
@@ -19,14 +19,11 @@
 
 package org.elasticsearch.test.rest;
 
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 
 import java.io.IOException;
 
-import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.startsWith;
@@ -49,26 +46,31 @@ public class CreatedLocationHeaderIT extends ESRestTestCase {
     }
 
     public void testUpsert() throws IOException {
-        locationTestCase(client().performRequest("POST", "test/test/1/_update", emptyMap(), new StringEntity("{"
-                + "\"doc\": {\"test\": \"test\"},"
-                + "\"doc_as_upsert\": true}", ContentType.APPLICATION_JSON)));
+        Request request = new Request("POST", "test/test/1/_update");
+        request.setJsonEntity("{"
+            + "\"doc\": {\"test\": \"test\"},"
+            + "\"doc_as_upsert\": true}");
+        locationTestCase(client().performRequest(request));
     }
 
     private void locationTestCase(String method, String url) throws IOException {
-        locationTestCase(client().performRequest(method, url, emptyMap(),
-            new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
+        final Request request = new Request(method, url);
+        request.setJsonEntity("{\"test\": \"test\"}");
+        locationTestCase(client().performRequest(request));
         // we have to delete the index otherwise the second indexing request will route to the single shard and not produce a 201
         final Response response = client().performRequest(new Request("DELETE", "test"));
         assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-        locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(),
-                new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
+        final Request withRouting = new Request(method, url);
+        withRouting.addParameter("routing", "cat");
+        withRouting.setJsonEntity("{\"test\": \"test\"}");
+        locationTestCase(client().performRequest(withRouting));
     }
 
     private void locationTestCase(Response response) throws IOException {
         assertEquals(201, response.getStatusLine().getStatusCode());
         String location = response.getHeader("Location");
         assertThat(location, startsWith("/test/test/"));
-        Response getResponse = client().performRequest("GET", location);
+        Response getResponse = client().performRequest(new Request("GET", location));
         assertEquals(singletonMap("test", "test"), entityAsMap(getResponse).get("_source"));
     }
 
diff --git a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java
index b94aa71b040..818037f68a1 100644
--- a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java
+++ b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java
@@ -19,13 +19,11 @@
 
 package org.elasticsearch.test.rest;
 
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.Request;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -39,8 +37,8 @@ public class NodeRestUsageIT extends ESRestTestCase {
     @SuppressWarnings("unchecked")
     public void testWithRestUsage() throws IOException {
         // First get the current usage figures
-        Response beforeResponse = client().performRequest("GET",
-                randomFrom("_nodes/usage", "_nodes/usage/rest_actions", "_nodes/usage/_all"));
+        String path = randomFrom("_nodes/usage", "_nodes/usage/rest_actions", "_nodes/usage/_all");
+        Response beforeResponse = client().performRequest(new Request("GET", path));
         Map<String, Object> beforeResponseBodyMap = entityAsMap(beforeResponse);
         assertThat(beforeResponseBodyMap, notNullValue());
         Map<String, Object> before_nodesMap = (Map<String, Object>) beforeResponseBodyMap.get("_nodes");
@@ -80,24 +78,24 @@ public class NodeRestUsageIT extends ESRestTestCase {
         }
 
         // Do some requests to get some rest usage stats
-        client().performRequest("PUT", "/test");
-        client().performRequest("POST", "/test/doc/1", Collections.emptyMap(),
-                new StringEntity("{ \"foo\": \"bar\"}", ContentType.APPLICATION_JSON));
-        client().performRequest("POST", "/test/doc/2", Collections.emptyMap(),
-                new StringEntity("{ \"foo\": \"bar\"}", ContentType.APPLICATION_JSON));
-        client().performRequest("POST", "/test/doc/3", Collections.emptyMap(),
-                new StringEntity("{ \"foo\": \"bar\"}", ContentType.APPLICATION_JSON));
-        client().performRequest("GET", "/test/_search");
-        client().performRequest("POST", "/test/doc/4", Collections.emptyMap(),
-                new StringEntity("{ \"foo\": \"bar\"}", ContentType.APPLICATION_JSON));
-        client().performRequest("POST", "/test/_refresh");
-        client().performRequest("GET", "/_cat/indices");
-        client().performRequest("GET", "/_nodes");
-        client().performRequest("GET", "/test/_search");
-        client().performRequest("GET", "/_nodes/stats");
-        client().performRequest("DELETE", "/test");
+        client().performRequest(new Request("PUT", "/test"));
+        for (int i = 0; i < 3; i++) {
+            final Request index = new Request("POST", "/test/doc/1");
+            index.setJsonEntity("{\"foo\": \"bar\"}");
+            client().performRequest(index);
+        }
+        client().performRequest(new Request("GET", "/test/_search"));
+        final Request index4 = new Request("POST", "/test/doc/4");
+        index4.setJsonEntity("{\"foo\": \"bar\"}");
+        client().performRequest(index4);
+        client().performRequest(new Request("POST", "/test/_refresh"));
+        client().performRequest(new Request("GET", "/_cat/indices"));
+        client().performRequest(new Request("GET", "/_nodes"));
+        client().performRequest(new Request("GET", "/test/_search"));
+        client().performRequest(new Request("GET", "/_nodes/stats"));
+        client().performRequest(new Request("DELETE", "/test"));
 
-        Response response = client().performRequest("GET", "_nodes/usage");
+        Response response = client().performRequest(new Request("GET", "_nodes/usage"));
         Map<String, Object> responseBodyMap = entityAsMap(response);
         assertThat(responseBodyMap, notNullValue());
         Map<String, Object> _nodesMap = (Map<String, Object>) responseBodyMap.get("_nodes");
@@ -139,7 +137,7 @@ public class NodeRestUsageIT extends ESRestTestCase {
 
     public void testMetricsWithAll() throws IOException {
         ResponseException exception = expectThrows(ResponseException.class,
-                () -> client().performRequest("GET", "_nodes/usage/_all,rest_actions"));
+                () -> client().performRequest(new Request("GET", "_nodes/usage/_all,rest_actions")));
         assertNotNull(exception);
         assertThat(exception.getMessage(), containsString("\"type\":\"illegal_argument_exception\","
                 + "\"reason\":\"request [_nodes/usage/_all,rest_actions] contains _all and individual metrics [_all,rest_actions]\""));
diff --git a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java
index ce72af26628..a6fc7b9cce1 100644
--- a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java
+++ b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.test.rest;
 
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.Request;
 
 import java.io.IOException;
 
@@ -28,56 +29,56 @@ import static org.hamcrest.CoreMatchers.containsString;
 public class RequestsWithoutContentIT extends ESRestTestCase {
 
     public void testIndexMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                randomBoolean() ? "POST" : "PUT", "/idx/type/123"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request(randomBoolean() ? "POST" : "PUT", "/idx/type/123")));
         assertResponseException(responseException, "request body is required");
     }
 
     public void testBulkMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                randomBoolean() ? "POST" : "PUT", "/_bulk"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request(randomBoolean() ? "POST" : "PUT", "/_bulk")));
         assertResponseException(responseException, "request body is required");
     }
 
     public void testPutSettingsMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                "PUT", "/_settings"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request("PUT", "/_settings")));
         assertResponseException(responseException, "request body is required");
     }
 
     public void testPutMappingsMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                randomBoolean() ? "POST" : "PUT", "/test_index/test_type/_mapping"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request(randomBoolean() ? "POST" : "PUT", "/test_index/test_type/_mapping")));
         assertResponseException(responseException, "request body is required");
     }
 
     public void testPutIndexTemplateMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                randomBoolean() ? "PUT" : "POST", "/_template/my_template"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request(randomBoolean() ? "PUT" : "POST", "/_template/my_template")));
         assertResponseException(responseException, "request body is required");
     }
 
     public void testMultiSearchMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                randomBoolean() ? "POST" : "GET", "/_msearch"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request(randomBoolean() ? "POST" : "GET", "/_msearch")));
         assertResponseException(responseException, "request body or source parameter is required");
     }
 
     public void testPutPipelineMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                "PUT", "/_ingest/pipeline/my_pipeline"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request("PUT", "/_ingest/pipeline/my_pipeline")));
         assertResponseException(responseException, "request body or source parameter is required");
     }
 
     public void testSimulatePipelineMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                randomBoolean() ? "POST" : "GET", "/_ingest/pipeline/my_pipeline/_simulate"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request(randomBoolean() ? "POST" : "GET", "/_ingest/pipeline/my_pipeline/_simulate")));
         assertResponseException(responseException, "request body or source parameter is required");
     }
 
     public void testPutScriptMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(
-                randomBoolean() ? "POST" : "PUT", "/_scripts/lang"));
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest(new Request(randomBoolean() ? "POST" : "PUT", "/_scripts/lang")));
         assertResponseException(responseException, "request body is required");
     }
 
diff --git a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java
index 0b1ad2a6dd9..fab809a51bc 100644
--- a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java
+++ b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java
@@ -19,26 +19,21 @@
 
 package org.elasticsearch.test.rest;
 
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.ResponseListener;
+import org.elasticsearch.client.Request;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 
-import static java.util.Collections.emptyMap;
-
 /**
  * Tests that wait for refresh is fired if the index is closed.
  */
@@ -46,13 +41,14 @@ public class WaitForRefreshAndCloseTests extends ESRestTestCase {
     @Before
     public void setupIndex() throws IOException {
         try {
-            client().performRequest("DELETE", indexName());
+            client().performRequest(new Request("DELETE", indexName()));
         } catch (ResponseException e) {
             // If we get an error, it should be because the index doesn't exist
             assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
         }
-        client().performRequest("PUT", indexName(), emptyMap(),
-            new StringEntity("{\"settings\":{\"refresh_interval\":-1}}", ContentType.APPLICATION_JSON));
+        Request request = new Request("PUT", indexName());
+        request.setJsonEntity("{\"settings\":{\"refresh_interval\":-1}}");
+        client().performRequest(request);
     }
 
     @After
@@ -69,17 +65,20 @@ public class WaitForRefreshAndCloseTests extends ESRestTestCase {
     }
 
     public void testIndexAndThenClose() throws Exception {
-        closeWhileListenerEngaged(start("PUT", "", new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON)));
+        closeWhileListenerEngaged(start("PUT", "", "{\"test\":\"test\"}"));
     }
 
     public void testUpdateAndThenClose() throws Exception {
-        client().performRequest("PUT", docPath(), emptyMap(), new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON));
-        closeWhileListenerEngaged(start("POST", "/_update",
-            new StringEntity("{\"doc\":{\"name\":\"test\"}}", ContentType.APPLICATION_JSON)));
+        Request request = new Request("PUT", docPath());
+        request.setJsonEntity("{\"test\":\"test\"}");
+        client().performRequest(request);
+        closeWhileListenerEngaged(start("POST", "/_update", "{\"doc\":{\"name\":\"test\"}}"));
     }
 
     public void testDeleteAndThenClose() throws Exception {
-        client().performRequest("PUT", docPath(), emptyMap(), new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON));
+        Request request = new Request("PUT", docPath());
+        request.setJsonEntity("{\"test\":\"test\"}");
+        client().performRequest(request);
         closeWhileListenerEngaged(start("DELETE", "", null));
     }
 
@@ -88,7 +87,7 @@ public class WaitForRefreshAndCloseTests extends ESRestTestCase {
         assertBusy(() -> {
             Map<String, Object> stats;
             try {
-                stats = entityAsMap(client().performRequest("GET", indexName() + "/_stats/refresh"));
+                stats = entityAsMap(client().performRequest(new Request("GET", indexName() + "/_stats/refresh")));
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
@@ -105,18 +104,19 @@ public class WaitForRefreshAndCloseTests extends ESRestTestCase {
         });
 
         // Close the index. That should flush the listener.
-        client().performRequest("POST", indexName() + "/_close");
+        client().performRequest(new Request("POST", indexName() + "/_close"));
 
         // The request shouldn't fail. It certainly shouldn't hang.
         future.get();
     }
 
-    private ActionFuture<String> start(String method, String path, HttpEntity body) {
+    private ActionFuture<String> start(String method, String path, String body) {
         PlainActionFuture<String> future = new PlainActionFuture<>();
-        Map<String, String> params = new HashMap<>();
-        params.put("refresh", "wait_for");
-        params.put("error_trace", "");
-        client().performRequestAsync(method, docPath() + path, params, body, new ResponseListener() {
+        Request request = new Request(method, docPath() + path);
+        request.addParameter("refresh", "wait_for");
+        request.addParameter("error_trace", "");
+        request.setJsonEntity(body);
+        client().performRequestAsync(request, new ResponseListener() {
             @Override
             public void onSuccess(Response response) {
                 try {

From df1380b8d3267d70771a640c7bac5a04fb63a8e1 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Tue, 17 Jul 2018 21:59:48 -0400
Subject: [PATCH 074/260] Remove versionType from translog (#31945)

With the introduction of sequence number, we no longer use versionType to
resolve out of order collision in replication and recovery requests.

This PR removes removes the versionType from translog. We can only remove
it in 7.0 because it is still required in a mixed cluster between 6.x and 5.x.
---
 .../action/bulk/TransportShardBulkAction.java |  5 +-
 .../org/elasticsearch/index/VersionType.java  | 15 -----
 .../elasticsearch/index/engine/Engine.java    |  2 +
 .../index/engine/InternalEngine.java          | 25 +-------
 .../elasticsearch/index/shard/IndexShard.java | 23 ++++---
 .../index/translog/Translog.java              | 61 ++++++++-----------
 .../index/translog/TranslogWriter.java        |  8 ++-
 .../resync/ResyncReplicationRequestTests.java |  4 +-
 .../index/engine/InternalEngineTests.java     | 30 ++++-----
 .../RecoveryDuringReplicationTests.java       |  1 -
 .../index/shard/IndexShardTests.java          | 27 ++++----
 .../index/translog/TranslogTests.java         | 25 ++++----
 .../indices/recovery/RecoveryTests.java       | 10 +--
 .../index/engine/EngineTestCase.java          |  9 +--
 .../index/engine/TranslogHandler.java         |  6 +-
 .../index/shard/IndexShardTestCase.java       |  4 +-
 16 files changed, 99 insertions(+), 156 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index a78421a2328..15a98077eac 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -523,13 +523,12 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                         indexRequest.type(), indexRequest.id(), indexRequest.source(), indexRequest.getContentType())
                         .routing(indexRequest.routing());
                 result = replica.applyIndexOperationOnReplica(primaryResponse.getSeqNo(), primaryResponse.getVersion(),
-                    indexRequest.versionType().versionTypeForReplicationAndRecovery(), indexRequest.getAutoGeneratedTimestamp(),
-                    indexRequest.isRetry(), sourceToParse);
+                    indexRequest.getAutoGeneratedTimestamp(), indexRequest.isRetry(), sourceToParse);
                 break;
             case DELETE:
                 DeleteRequest deleteRequest = (DeleteRequest) docWriteRequest;
                 result =  replica.applyDeleteOperationOnReplica(primaryResponse.getSeqNo(), primaryResponse.getVersion(),
-                    deleteRequest.type(), deleteRequest.id(), deleteRequest.versionType().versionTypeForReplicationAndRecovery());
+                    deleteRequest.type(), deleteRequest.id());
                 break;
             default:
                 throw new IllegalStateException("Unexpected request operation type on replica: "
diff --git a/server/src/main/java/org/elasticsearch/index/VersionType.java b/server/src/main/java/org/elasticsearch/index/VersionType.java
index b350252dc9c..eb05ff80907 100644
--- a/server/src/main/java/org/elasticsearch/index/VersionType.java
+++ b/server/src/main/java/org/elasticsearch/index/VersionType.java
@@ -85,13 +85,6 @@ public enum VersionType implements Writeable {
             // not allowing Versions.NOT_FOUND as it is not a valid input value.
             return version > 0L || version == Versions.MATCH_ANY;
         }
-
-        @Override
-        public VersionType versionTypeForReplicationAndRecovery() {
-            // replicas get the version from the primary after increment. The same version is stored in
-            // the transaction log. -> the should use the external semantics.
-            return EXTERNAL;
-        }
     },
     EXTERNAL((byte) 1) {
         @Override
@@ -333,14 +326,6 @@ public enum VersionType implements Writeable {
      */
     public abstract boolean validateVersionForReads(long version);
 
-    /**
-     * Some version types require different semantics for primary and replicas. This version allows
-     * the type to override the default behavior.
-     */
-    public VersionType versionTypeForReplicationAndRecovery() {
-        return this;
-    }
-
     public static VersionType fromString(String versionType) {
         if ("internal".equals(versionType)) {
             return INTERNAL;
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index 8a560e02fe4..53a7baa60f6 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -1168,6 +1168,7 @@ public abstract class Engine implements Closeable {
         public Index(Term uid, ParsedDocument doc, long seqNo, long primaryTerm, long version, VersionType versionType, Origin origin,
                      long startTime, long autoGeneratedIdTimestamp, boolean isRetry) {
             super(uid, seqNo, primaryTerm, version, versionType, origin, startTime);
+            assert (origin == Origin.PRIMARY) == (versionType != null) : "invalid version_type=" + versionType + " for origin=" + origin;
             this.doc = doc;
             this.isRetry = isRetry;
             this.autoGeneratedIdTimestamp = autoGeneratedIdTimestamp;
@@ -1245,6 +1246,7 @@ public abstract class Engine implements Closeable {
         public Delete(String type, String id, Term uid, long seqNo, long primaryTerm, long version, VersionType versionType,
                       Origin origin, long startTime) {
             super(uid, seqNo, primaryTerm, version, versionType, origin, startTime);
+            assert (origin == Origin.PRIMARY) == (versionType != null) : "invalid version_type=" + versionType + " for origin=" + origin;
             this.type = Objects.requireNonNull(type);
             this.id = Objects.requireNonNull(id);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 92c64d415ad..bdcfb2fc731 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -691,7 +691,7 @@ public class InternalEngine extends Engine {
                     return true;
                 case PEER_RECOVERY:
                 case REPLICA:
-                    assert index.version() == 1 && index.versionType() == VersionType.EXTERNAL
+                    assert index.version() == 1 && index.versionType() == null
                         : "version: " + index.version() + " type: " + index.versionType();
                     return true;
                 case LOCAL_TRANSLOG_RECOVERY:
@@ -704,20 +704,6 @@ public class InternalEngine extends Engine {
         return false;
     }
 
-    private boolean assertVersionType(final Engine.Operation operation) {
-        if (operation.origin() == Operation.Origin.REPLICA ||
-                operation.origin() == Operation.Origin.PEER_RECOVERY ||
-                operation.origin() == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
-            // ensure that replica operation has expected version type for replication
-            // ensure that versionTypeForReplicationAndRecovery is idempotent
-            assert operation.versionType() == operation.versionType().versionTypeForReplicationAndRecovery()
-                    : "unexpected version type in request from [" + operation.origin().name() + "] " +
-                    "found [" + operation.versionType().name() + "] " +
-                    "expected [" + operation.versionType().versionTypeForReplicationAndRecovery().name() + "]";
-        }
-        return true;
-    }
-
     private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) {
         if (origin == Operation.Origin.PRIMARY) {
             assert assertOriginPrimarySequenceNumber(seqNo);
@@ -757,7 +743,6 @@ public class InternalEngine extends Engine {
         try (ReleasableLock releasableLock = readLock.acquire()) {
             ensureOpen();
             assert assertIncomingSequenceNumber(index.origin(), index.seqNo());
-            assert assertVersionType(index);
             try (Releasable ignored = versionMap.acquireLock(index.uid().bytes());
                 Releasable indexThrottle = doThrottle ? () -> {} : throttle.acquireThrottle()) {
                 lastWriteNanos = index.startTime();
@@ -860,9 +845,6 @@ public class InternalEngine extends Engine {
                     "max_seqno non-append-only [" + maxSeqNoOfNonAppendOnlyOperations.get() + "], seqno of index [" + index.seqNo() + "]";
             }
             versionMap.enforceSafeAccess();
-            // drop out of order operations
-            assert index.versionType().versionTypeForReplicationAndRecovery() == index.versionType() :
-                "resolving out of order delivery based on versioning but version type isn't fit for it. got [" + index.versionType() + "]";
             // unlike the primary, replicas don't really care to about creation status of documents
             // this allows to ignore the case where a document was found in the live version maps in
             // a delete state and return false for the created flag in favor of code simplicity
@@ -1096,7 +1078,6 @@ public class InternalEngine extends Engine {
     public DeleteResult delete(Delete delete) throws IOException {
         versionMap.enforceSafeAccess();
         assert Objects.equals(delete.uid().field(), IdFieldMapper.NAME) : delete.uid().field();
-        assert assertVersionType(delete);
         assert assertIncomingSequenceNumber(delete.origin(), delete.seqNo());
         final DeleteResult deleteResult;
         // NOTE: we don't throttle this when merges fall behind because delete-by-id does not create new segments:
@@ -1149,10 +1130,6 @@ public class InternalEngine extends Engine {
 
     private DeletionStrategy planDeletionAsNonPrimary(Delete delete) throws IOException {
         assert delete.origin() != Operation.Origin.PRIMARY : "planing as primary but got " + delete.origin();
-        // drop out of order operations
-        assert delete.versionType().versionTypeForReplicationAndRecovery() == delete.versionType() :
-            "resolving out of order delivery based on versioning but version type isn't fit for it. got ["
-                + delete.versionType() + "]";
         maxSeqNoOfNonAppendOnlyOperations.updateAndGet(curr -> Math.max(delete.seqNo(), curr));
         assert maxSeqNoOfNonAppendOnlyOperations.get() >= delete.seqNo() : "max_seqno of non-append-only was not updated;" +
             "max_seqno non-append-only [" + maxSeqNoOfNonAppendOnlyOperations.get() + "], seqno of delete [" + delete.seqNo() + "]";
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 5bd8f9abc6e..b07e22875e8 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -645,22 +645,22 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
 
     public Engine.IndexResult applyIndexOperationOnPrimary(long version, VersionType versionType, SourceToParse sourceToParse,
                                                            long autoGeneratedTimestamp, boolean isRetry) throws IOException {
+        assert versionType.validateVersionForWrites(version);
         return applyIndexOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm, version, versionType, autoGeneratedTimestamp,
             isRetry, Engine.Operation.Origin.PRIMARY, sourceToParse);
     }
 
-    public Engine.IndexResult applyIndexOperationOnReplica(long seqNo, long version, VersionType versionType,
-                                                           long autoGeneratedTimeStamp, boolean isRetry, SourceToParse sourceToParse)
+    public Engine.IndexResult applyIndexOperationOnReplica(long seqNo, long version, long autoGeneratedTimeStamp,
+                                                           boolean isRetry, SourceToParse sourceToParse)
         throws IOException {
-        return applyIndexOperation(seqNo, primaryTerm, version, versionType, autoGeneratedTimeStamp, isRetry,
+        return applyIndexOperation(seqNo, primaryTerm, version, null, autoGeneratedTimeStamp, isRetry,
             Engine.Operation.Origin.REPLICA, sourceToParse);
     }
 
-    private Engine.IndexResult applyIndexOperation(long seqNo, long opPrimaryTerm, long version, VersionType versionType,
+    private Engine.IndexResult applyIndexOperation(long seqNo, long opPrimaryTerm, long version, @Nullable VersionType versionType,
                                                    long autoGeneratedTimeStamp, boolean isRetry, Engine.Operation.Origin origin,
                                                    SourceToParse sourceToParse) throws IOException {
         assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]";
-        assert versionType.validateVersionForWrites(version);
         ensureWriteAllowed(origin);
         Engine.Index operation;
         try {
@@ -736,19 +736,18 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
 
     public Engine.DeleteResult applyDeleteOperationOnPrimary(long version, String type, String id, VersionType versionType)
         throws IOException {
+        assert versionType.validateVersionForWrites(version);
         return applyDeleteOperation(SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm, version, type, id, versionType,
             Engine.Operation.Origin.PRIMARY);
     }
 
-    public Engine.DeleteResult applyDeleteOperationOnReplica(long seqNo, long version, String type, String id,
-                                                             VersionType versionType) throws IOException {
-        return applyDeleteOperation(seqNo, primaryTerm, version, type, id, versionType, Engine.Operation.Origin.REPLICA);
+    public Engine.DeleteResult applyDeleteOperationOnReplica(long seqNo, long version, String type, String id) throws IOException {
+        return applyDeleteOperation(seqNo, primaryTerm, version, type, id, null, Engine.Operation.Origin.REPLICA);
     }
 
     private Engine.DeleteResult applyDeleteOperation(long seqNo, long opPrimaryTerm, long version, String type, String id,
-                                                     VersionType versionType, Engine.Operation.Origin origin) throws IOException {
+                                                     @Nullable VersionType versionType, Engine.Operation.Origin origin) throws IOException {
         assert opPrimaryTerm <= this.primaryTerm : "op term [ " + opPrimaryTerm + " ] > shard term [" + this.primaryTerm + "]";
-        assert versionType.validateVersionForWrites(version);
         ensureWriteAllowed(origin);
         // When there is a single type, the unique identifier is only composed of the _id,
         // so there is no way to differenciate foo#1 from bar#1. This is especially an issue
@@ -1211,14 +1210,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                 // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all
                 // autoGeneratedID docs that are coming from the primary are updated correctly.
                 result = applyIndexOperation(index.seqNo(), index.primaryTerm(), index.version(),
-                    index.versionType().versionTypeForReplicationAndRecovery(), index.getAutoGeneratedIdTimestamp(), true, origin,
+                    null, index.getAutoGeneratedIdTimestamp(), true, origin,
                     source(shardId.getIndexName(), index.type(), index.id(), index.source(),
                         XContentHelper.xContentType(index.source())).routing(index.routing()));
                 break;
             case DELETE:
                 final Translog.Delete delete = (Translog.Delete) operation;
                 result = applyDeleteOperation(delete.seqNo(), delete.primaryTerm(), delete.version(), delete.type(), delete.id(),
-                    delete.versionType().versionTypeForReplicationAndRecovery(), origin);
+                    null, origin);
                 break;
             case NO_OP:
                 final Translog.NoOp noOp = (Translog.NoOp) operation;
diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
index 63055d933e4..31404b7874a 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -1011,7 +1011,8 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
 
         public static final int FORMAT_6_0 = 8; // since 6.0.0
         public static final int FORMAT_NO_PARENT = FORMAT_6_0 + 1; // since 7.0
-        public static final int SERIALIZATION_FORMAT = FORMAT_NO_PARENT;
+        public static final int FORMAT_NO_VERSION_TYPE = FORMAT_NO_PARENT + 1;
+        public static final int SERIALIZATION_FORMAT = FORMAT_NO_VERSION_TYPE;
 
         private final String id;
         private final long autoGeneratedIdTimestamp;
@@ -1019,7 +1020,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
         private final long seqNo;
         private final long primaryTerm;
         private final long version;
-        private final VersionType versionType;
         private final BytesReference source;
         private final String routing;
 
@@ -1034,8 +1034,9 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                 in.readOptionalString(); // _parent
             }
             this.version = in.readLong();
-            this.versionType = VersionType.fromValue(in.readByte());
-            assert versionType.validateVersionForWrites(this.version) : "invalid version for writes: " + this.version;
+            if (format < FORMAT_NO_VERSION_TYPE) {
+                in.readByte(); // _version_type
+            }
             this.autoGeneratedIdTimestamp = in.readLong();
             seqNo = in.readLong();
             primaryTerm = in.readLong();
@@ -1049,15 +1050,14 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             this.seqNo = indexResult.getSeqNo();
             this.primaryTerm = index.primaryTerm();
             this.version = indexResult.getVersion();
-            this.versionType = index.versionType();
             this.autoGeneratedIdTimestamp = index.getAutoGeneratedIdTimestamp();
         }
 
         public Index(String type, String id, long seqNo, long primaryTerm, byte[] source) {
-            this(type, id, seqNo, primaryTerm, Versions.MATCH_ANY, VersionType.INTERNAL, source, null, -1);
+            this(type, id, seqNo, primaryTerm, Versions.MATCH_ANY, source, null, -1);
         }
 
-        public Index(String type, String id, long seqNo, long primaryTerm, long version, VersionType versionType,
+        public Index(String type, String id, long seqNo, long primaryTerm, long version,
                      byte[] source, String routing, long autoGeneratedIdTimestamp) {
             this.type = type;
             this.id = id;
@@ -1065,7 +1065,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             this.seqNo = seqNo;
             this.primaryTerm = primaryTerm;
             this.version = version;
-            this.versionType = versionType;
             this.routing = routing;
             this.autoGeneratedIdTimestamp = autoGeneratedIdTimestamp;
         }
@@ -1110,24 +1109,22 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             return this.version;
         }
 
-        public VersionType versionType() {
-            return versionType;
-        }
-
         @Override
         public Source getSource() {
             return new Source(source, routing);
         }
 
         private void write(final StreamOutput out) throws IOException {
-            out.writeVInt(SERIALIZATION_FORMAT);
+            final int format = out.getVersion().onOrAfter(Version.V_7_0_0_alpha1) ? SERIALIZATION_FORMAT : FORMAT_6_0;
+            out.writeVInt(format);
             out.writeString(id);
             out.writeString(type);
             out.writeBytesReference(source);
             out.writeOptionalString(routing);
             out.writeLong(version);
-
-            out.writeByte(versionType.getValue());
+            if (format < FORMAT_NO_VERSION_TYPE) {
+                out.writeByte(VersionType.EXTERNAL.getValue());
+            }
             out.writeLong(autoGeneratedIdTimestamp);
             out.writeLong(seqNo);
             out.writeLong(primaryTerm);
@@ -1149,7 +1146,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                 primaryTerm != index.primaryTerm ||
                 id.equals(index.id) == false ||
                 type.equals(index.type) == false ||
-                versionType != index.versionType ||
                 autoGeneratedIdTimestamp != index.autoGeneratedIdTimestamp ||
                 source.equals(index.source) == false) {
                 return false;
@@ -1168,7 +1164,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             result = 31 * result + Long.hashCode(seqNo);
             result = 31 * result + Long.hashCode(primaryTerm);
             result = 31 * result + Long.hashCode(version);
-            result = 31 * result + versionType.hashCode();
             result = 31 * result + source.hashCode();
             result = 31 * result + (routing != null ? routing.hashCode() : 0);
             result = 31 * result + Long.hashCode(autoGeneratedIdTimestamp);
@@ -1194,14 +1189,15 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     public static class Delete implements Operation {
 
         private static final int FORMAT_6_0 = 4; // 6.0 - *
-        public static final int SERIALIZATION_FORMAT = FORMAT_6_0;
+        public static final int FORMAT_NO_PARENT = FORMAT_6_0 + 1; // since 7.0
+        public static final int FORMAT_NO_VERSION_TYPE = FORMAT_NO_PARENT + 1;
+        public static final int SERIALIZATION_FORMAT = FORMAT_NO_VERSION_TYPE;
 
         private final String type, id;
         private final Term uid;
         private final long seqNo;
         private final long primaryTerm;
         private final long version;
-        private final VersionType versionType;
 
         private Delete(final StreamInput in) throws IOException {
             final int format = in.readVInt();// SERIALIZATION_FORMAT
@@ -1210,29 +1206,29 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             id = in.readString();
             uid = new Term(in.readString(), in.readBytesRef());
             this.version = in.readLong();
-            this.versionType = VersionType.fromValue(in.readByte());
-            assert versionType.validateVersionForWrites(this.version);
+            if (format < FORMAT_NO_VERSION_TYPE) {
+                in.readByte(); // versionType
+            }
             seqNo = in.readLong();
             primaryTerm = in.readLong();
         }
 
         public Delete(Engine.Delete delete, Engine.DeleteResult deleteResult) {
-            this(delete.type(), delete.id(), delete.uid(), deleteResult.getSeqNo(), delete.primaryTerm(), deleteResult.getVersion(), delete.versionType());
+            this(delete.type(), delete.id(), delete.uid(), deleteResult.getSeqNo(), delete.primaryTerm(), deleteResult.getVersion());
         }
 
         /** utility for testing */
         public Delete(String type, String id, long seqNo, long primaryTerm, Term uid) {
-            this(type, id, uid, seqNo, primaryTerm, Versions.MATCH_ANY, VersionType.INTERNAL);
+            this(type, id, uid, seqNo, primaryTerm, Versions.MATCH_ANY);
         }
 
-        public Delete(String type, String id, Term uid, long seqNo, long primaryTerm, long version, VersionType versionType) {
+        public Delete(String type, String id, Term uid, long seqNo, long primaryTerm, long version) {
             this.type = Objects.requireNonNull(type);
             this.id = Objects.requireNonNull(id);
             this.uid = uid;
             this.seqNo = seqNo;
             this.primaryTerm = primaryTerm;
             this.version = version;
-            this.versionType = versionType;
         }
 
         @Override
@@ -1271,23 +1267,22 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             return this.version;
         }
 
-        public VersionType versionType() {
-            return this.versionType;
-        }
-
         @Override
         public Source getSource() {
             throw new IllegalStateException("trying to read doc source from delete operation");
         }
 
         private void write(final StreamOutput out) throws IOException {
-            out.writeVInt(SERIALIZATION_FORMAT);
+            final int format = out.getVersion().onOrAfter(Version.V_7_0_0_alpha1) ? SERIALIZATION_FORMAT : FORMAT_6_0;
+            out.writeVInt(format);
             out.writeString(type);
             out.writeString(id);
             out.writeString(uid.field());
             out.writeBytesRef(uid.bytes());
             out.writeLong(version);
-            out.writeByte(versionType.getValue());
+            if (format < FORMAT_NO_VERSION_TYPE) {
+                out.writeByte(VersionType.EXTERNAL.getValue());
+            }
             out.writeLong(seqNo);
             out.writeLong(primaryTerm);
         }
@@ -1306,8 +1301,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             return version == delete.version &&
                 seqNo == delete.seqNo &&
                 primaryTerm == delete.primaryTerm &&
-                uid.equals(delete.uid) &&
-                versionType == delete.versionType;
+                uid.equals(delete.uid);
         }
 
         @Override
@@ -1316,7 +1310,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             result = 31 * result + Long.hashCode(seqNo);
             result = 31 * result + Long.hashCode(primaryTerm);
             result = 31 * result + Long.hashCode(version);
-            result = 31 * result + versionType.hashCode();
             return result;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index b89b21c5258..c135facc67f 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -202,9 +202,11 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
             if (previous.v1().equals(data) == false) {
                 Translog.Operation newOp = Translog.readOperation(new BufferedChecksumStreamInput(data.streamInput()));
                 Translog.Operation prvOp = Translog.readOperation(new BufferedChecksumStreamInput(previous.v1().streamInput()));
-                throw new AssertionError(
-                    "seqNo [" + seqNo + "] was processed twice in generation [" + generation + "], with different data. " +
-                        "prvOp [" + prvOp + "], newOp [" + newOp + "]", previous.v2());
+                if (newOp.equals(prvOp) == false) {
+                    throw new AssertionError(
+                        "seqNo [" + seqNo + "] was processed twice in generation [" + generation + "], with different data. " +
+                            "prvOp [" + prvOp + "], newOp [" + newOp + "]", previous.v2());
+                }
             }
         } else {
             seenSequenceNumbers.put(seqNo,
diff --git a/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java b/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java
index 914c2b87422..15b8e1c99d2 100644
--- a/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/resync/ResyncReplicationRequestTests.java
@@ -21,9 +21,7 @@ package org.elasticsearch.action.resync;
 
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.test.ESTestCase;
@@ -38,7 +36,7 @@ public class ResyncReplicationRequestTests extends ESTestCase {
     public void testSerialization() throws IOException {
         final byte[] bytes = "{}".getBytes(Charset.forName("UTF-8"));
         final Translog.Index index = new Translog.Index("type", "id", 0, randomNonNegativeLong(),
-            Versions.MATCH_ANY, VersionType.INTERNAL, bytes, null, -1);
+            randomNonNegativeLong(), bytes, null, -1);
         final ShardId shardId = new ShardId(new Index("index", "uuid"), 0);
         final ResyncReplicationRequest before = new ResyncReplicationRequest(shardId, 42L, new Translog.Operation[]{index});
 
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 2e89a66805c..87b63dfdef8 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -1183,7 +1183,7 @@ public class InternalEngineTests extends EngineTestCase {
         assertThat(indexResult.getVersion(), equalTo(1L));
 
         create = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), create.primaryTerm(), indexResult.getVersion(),
-            create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+            null, REPLICA, 0, -1, false);
         indexResult = replicaEngine.index(create);
         assertThat(indexResult.getVersion(), equalTo(1L));
     }
@@ -1197,7 +1197,7 @@ public class InternalEngineTests extends EngineTestCase {
 
 
         create = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), create.primaryTerm(), indexResult.getVersion(),
-            create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+            null, REPLICA, 0, -1, false);
         indexResult = replicaEngine.index(create);
         assertThat(indexResult.getVersion(), equalTo(1L));
         assertTrue(indexResult.isCreated());
@@ -1216,7 +1216,7 @@ public class InternalEngineTests extends EngineTestCase {
 
 
         update = new Engine.Index(newUid(doc), doc, updateResult.getSeqNo(), update.primaryTerm(), updateResult.getVersion(),
-            update.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+            null, REPLICA, 0, -1, false);
         updateResult = replicaEngine.index(update);
         assertThat(updateResult.getVersion(), equalTo(2L));
         assertFalse(updateResult.isCreated());
@@ -1269,7 +1269,7 @@ public class InternalEngineTests extends EngineTestCase {
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), null, REPLICA, 0, -1, false);
         indexResult = replicaEngine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
     }
@@ -1418,7 +1418,7 @@ public class InternalEngineTests extends EngineTestCase {
                     forReplica && i >= startWithSeqNo ? i * 2 : SequenceNumbers.UNASSIGNED_SEQ_NO,
                     forReplica && i >= startWithSeqNo && incrementTermWhenIntroducingSeqNo ? primaryTerm + 1 : primaryTerm,
                     version,
-                    forReplica ? versionType.versionTypeForReplicationAndRecovery() : versionType,
+                    forReplica ? null : versionType,
                     forReplica ? REPLICA : PRIMARY,
                     System.currentTimeMillis(), -1, false
                 );
@@ -1427,7 +1427,7 @@ public class InternalEngineTests extends EngineTestCase {
                     forReplica && i >= startWithSeqNo ? i * 2 : SequenceNumbers.UNASSIGNED_SEQ_NO,
                     forReplica && i >= startWithSeqNo && incrementTermWhenIntroducingSeqNo ? primaryTerm + 1 : primaryTerm,
                     version,
-                    forReplica ? versionType.versionTypeForReplicationAndRecovery() : versionType,
+                    forReplica ? null : versionType,
                     forReplica ? REPLICA : PRIMARY,
                     System.currentTimeMillis());
             }
@@ -3221,7 +3221,7 @@ public class InternalEngineTests extends EngineTestCase {
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), null, REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         indexResult = replicaEngine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
@@ -3235,7 +3235,7 @@ public class InternalEngineTests extends EngineTestCase {
             assertEquals(1, topDocs.totalHits);
         }
 
-        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), null, REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         indexResult = replicaEngine.index(index);
         assertThat(indexResult.getResultType(), equalTo(Engine.Result.Type.SUCCESS));
         replicaEngine.refresh("test");
@@ -3255,7 +3255,7 @@ public class InternalEngineTests extends EngineTestCase {
         Engine.IndexResult result = engine.index(firstIndexRequest);
         assertThat(result.getVersion(), equalTo(1L));
 
-        Engine.Index firstIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), firstIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.Index firstIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), firstIndexRequest.primaryTerm(), result.getVersion(), null, REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         Engine.IndexResult indexReplicaResult = replicaEngine.index(firstIndexRequestReplica);
         assertThat(indexReplicaResult.getVersion(), equalTo(1L));
 
@@ -3269,7 +3269,7 @@ public class InternalEngineTests extends EngineTestCase {
             assertEquals(1, topDocs.totalHits);
         }
 
-        Engine.Index secondIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), secondIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.Index secondIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), secondIndexRequest.primaryTerm(), result.getVersion(), null, REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         replicaEngine.index(secondIndexRequestReplica);
         replicaEngine.refresh("test");
         try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
@@ -3292,7 +3292,7 @@ public class InternalEngineTests extends EngineTestCase {
     }
 
     public Engine.Index appendOnlyReplica(ParsedDocument doc, boolean retry, final long autoGeneratedIdTimestamp, final long seqNo) {
-        return new Engine.Index(newUid(doc), doc, seqNo, 2, 1, VersionType.EXTERNAL,
+        return new Engine.Index(newUid(doc), doc, seqNo, 2, 1, null,
             Engine.Operation.Origin.REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, retry);
     }
 
@@ -3694,7 +3694,7 @@ public class InternalEngineTests extends EngineTestCase {
                     sequenceNumberSupplier.getAsLong(),
                     1,
                     i,
-                    VersionType.EXTERNAL,
+                    origin == PRIMARY ? VersionType.EXTERNAL : null,
                     origin,
                     System.nanoTime(),
                     IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP,
@@ -3708,7 +3708,7 @@ public class InternalEngineTests extends EngineTestCase {
                     sequenceNumberSupplier.getAsLong(),
                     1,
                     i,
-                    VersionType.EXTERNAL,
+                    origin == PRIMARY ? VersionType.EXTERNAL : null,
                     origin,
                     System.nanoTime());
                 operations.add(delete);
@@ -3928,7 +3928,7 @@ public class InternalEngineTests extends EngineTestCase {
                 final ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
                 final Term uid = newUid(doc);
                 final long time = System.nanoTime();
-                actualEngine.index(new Engine.Index(uid, doc, seqNo, 1, 1, VersionType.EXTERNAL, REPLICA, time, time, false));
+                actualEngine.index(new Engine.Index(uid, doc, seqNo, 1, 1, null, REPLICA, time, time, false));
                 if (rarely()) {
                     actualEngine.rollTranslogGeneration();
                 }
@@ -4686,7 +4686,7 @@ public class InternalEngineTests extends EngineTestCase {
                 for (int i = 0; i < seqNos.size(); i++) {
                     ParsedDocument doc = testParsedDocument(Long.toString(seqNos.get(i)), null, testDocument(), new BytesArray("{}"), null);
                     Engine.Index index = new Engine.Index(newUid(doc), doc, seqNos.get(i), 0,
-                        1, VersionType.EXTERNAL, REPLICA, System.nanoTime(), -1, false);
+                        1, null, REPLICA, System.nanoTime(), -1, false);
                     engine.index(index);
                     if (randomBoolean()) {
                         engine.flush();
diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
index ee97ba14fe0..f01d5e54a2e 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -186,7 +186,6 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC
             remainingReplica.applyIndexOperationOnReplica(
                     remainingReplica.getLocalCheckpoint() + 1,
                     1,
-                    VersionType.EXTERNAL,
                     randomNonNegativeLong(),
                     false,
                     SourceToParse.source("index", "type", "replica", new BytesArray("{}"), XContentType.JSON));
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index ac52378fc6b..2e07ec950a5 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -72,7 +72,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineException;
@@ -1545,17 +1544,17 @@ public class IndexShardTests extends IndexShardTestCase {
          * - If flush and then recover from the existing store, delete #1 will be removed while index #0 is still retained and replayed.
          */
         final IndexShard shard = newStartedShard(false);
-        shard.applyDeleteOperationOnReplica(1, 2, "_doc", "id", VersionType.EXTERNAL);
+        shard.applyDeleteOperationOnReplica(1, 2, "_doc", "id");
         shard.getEngine().rollTranslogGeneration(); // isolate the delete in it's own generation
-        shard.applyIndexOperationOnReplica(0, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+        shard.applyIndexOperationOnReplica(0, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             SourceToParse.source(shard.shardId().getIndexName(), "_doc", "id", new BytesArray("{}"), XContentType.JSON));
-        shard.applyIndexOperationOnReplica(3, 3, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+        shard.applyIndexOperationOnReplica(3, 3, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             SourceToParse.source(shard.shardId().getIndexName(), "_doc", "id-3", new BytesArray("{}"), XContentType.JSON));
         // Flushing a new commit with local checkpoint=1 allows to skip the translog gen #1 in recovery.
         shard.flush(new FlushRequest().force(true).waitIfOngoing(true));
-        shard.applyIndexOperationOnReplica(2, 3, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+        shard.applyIndexOperationOnReplica(2, 3, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             SourceToParse.source(shard.shardId().getIndexName(), "_doc", "id-2", new BytesArray("{}"), XContentType.JSON));
-        shard.applyIndexOperationOnReplica(5, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+        shard.applyIndexOperationOnReplica(5, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             SourceToParse.source(shard.shardId().getIndexName(), "_doc", "id-5", new BytesArray("{}"), XContentType.JSON));
 
         final int translogOps;
@@ -1646,8 +1645,7 @@ public class IndexShardTests extends IndexShardTestCase {
         updateMappings(otherShard, shard.indexSettings().getIndexMetaData());
         SourceToParse sourceToParse = SourceToParse.source(shard.shardId().getIndexName(), "_doc", "1",
             new BytesArray("{}"), XContentType.JSON);
-        otherShard.applyIndexOperationOnReplica(1, 1,
-            VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, sourceToParse);
+        otherShard.applyIndexOperationOnReplica(1, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, sourceToParse);
 
         final ShardRouting primaryShardRouting = shard.routingEntry();
         IndexShard newShard = reinitShard(otherShard, ShardRoutingHelper.initWithSameId(primaryShardRouting,
@@ -1763,18 +1761,18 @@ public class IndexShardTests extends IndexShardTestCase {
         final IndexShard shard = newStartedShard(false);
         final String indexName = shard.shardId().getIndexName();
         // Index #0, index #1
-        shard.applyIndexOperationOnReplica(0, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+        shard.applyIndexOperationOnReplica(0, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             SourceToParse.source(indexName, "_doc", "doc-0", new BytesArray("{}"), XContentType.JSON));
         flushShard(shard);
         shard.updateGlobalCheckpointOnReplica(0, "test"); // stick the global checkpoint here.
-        shard.applyIndexOperationOnReplica(1, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+        shard.applyIndexOperationOnReplica(1, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             SourceToParse.source(indexName, "_doc", "doc-1", new BytesArray("{}"), XContentType.JSON));
         flushShard(shard);
         assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1"));
         // Simulate resync (without rollback): Noop #1, index #2
         acquireReplicaOperationPermitBlockingly(shard, shard.primaryTerm + 1);
         shard.markSeqNoAsNoop(1, "test");
-        shard.applyIndexOperationOnReplica(2, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+        shard.applyIndexOperationOnReplica(2, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             SourceToParse.source(indexName, "_doc", "doc-2", new BytesArray("{}"), XContentType.JSON));
         flushShard(shard);
         assertThat(getShardDocUIDs(shard), containsInAnyOrder("doc-0", "doc-1", "doc-2"));
@@ -2104,11 +2102,11 @@ public class IndexShardTests extends IndexShardTestCase {
         int numCorruptEntries = 0;
         for (int i = 0; i < numTotalEntries; i++) {
             if (randomBoolean()) {
-                operations.add(new Translog.Index("_doc", "1", 0, primary.getPrimaryTerm(), 1, VersionType.INTERNAL,
+                operations.add(new Translog.Index("_doc", "1", 0, primary.getPrimaryTerm(), 1,
                     "{\"foo\" : \"bar\"}".getBytes(Charset.forName("UTF-8")), null, -1));
             } else {
                 // corrupt entry
-                operations.add(new Translog.Index("_doc", "2", 1,  primary.getPrimaryTerm(), 1, VersionType.INTERNAL,
+                operations.add(new Translog.Index("_doc", "2", 1,  primary.getPrimaryTerm(), 1,
                     "{\"foo\" : \"bar}".getBytes(Charset.forName("UTF-8")), null, -1));
                 numCorruptEntries++;
             }
@@ -2603,8 +2601,7 @@ public class IndexShardTests extends IndexShardTestCase {
                 final String id = Integer.toString(i);
                 SourceToParse sourceToParse = SourceToParse.source(indexShard.shardId().getIndexName(), "_doc", id,
                         new BytesArray("{}"), XContentType.JSON);
-                indexShard.applyIndexOperationOnReplica(i,
-                    1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, sourceToParse);
+                indexShard.applyIndexOperationOnReplica(i, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, sourceToParse);
                 if (!gap && i == localCheckpoint + 1) {
                     localCheckpoint++;
                 }
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index dbbb38090bc..b255238c864 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -416,9 +416,9 @@ public class TranslogTests extends ESTestCase {
         {
             final TranslogStats stats = stats();
             assertThat(stats.estimatedNumberOfOperations(), equalTo(1));
-            assertThat(stats.getTranslogSizeInBytes(), equalTo(163L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(162L));
             assertThat(stats.getUncommittedOperations(), equalTo(1));
-            assertThat(stats.getUncommittedSizeInBytes(), equalTo(163L));
+            assertThat(stats.getUncommittedSizeInBytes(), equalTo(162L));
             assertThat(stats.getEarliestLastModifiedAge(), greaterThan(1L));
         }
 
@@ -426,9 +426,9 @@ public class TranslogTests extends ESTestCase {
         {
             final TranslogStats stats = stats();
             assertThat(stats.estimatedNumberOfOperations(), equalTo(2));
-            assertThat(stats.getTranslogSizeInBytes(), equalTo(212L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(210L));
             assertThat(stats.getUncommittedOperations(), equalTo(2));
-            assertThat(stats.getUncommittedSizeInBytes(), equalTo(212L));
+            assertThat(stats.getUncommittedSizeInBytes(), equalTo(210L));
             assertThat(stats.getEarliestLastModifiedAge(), greaterThan(1L));
         }
 
@@ -436,9 +436,9 @@ public class TranslogTests extends ESTestCase {
         {
             final TranslogStats stats = stats();
             assertThat(stats.estimatedNumberOfOperations(), equalTo(3));
-            assertThat(stats.getTranslogSizeInBytes(), equalTo(261L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(258L));
             assertThat(stats.getUncommittedOperations(), equalTo(3));
-            assertThat(stats.getUncommittedSizeInBytes(), equalTo(261L));
+            assertThat(stats.getUncommittedSizeInBytes(), equalTo(258L));
             assertThat(stats.getEarliestLastModifiedAge(), greaterThan(1L));
         }
 
@@ -446,13 +446,13 @@ public class TranslogTests extends ESTestCase {
         {
             final TranslogStats stats = stats();
             assertThat(stats.estimatedNumberOfOperations(), equalTo(4));
-            assertThat(stats.getTranslogSizeInBytes(), equalTo(303L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(300L));
             assertThat(stats.getUncommittedOperations(), equalTo(4));
-            assertThat(stats.getUncommittedSizeInBytes(), equalTo(303L));
+            assertThat(stats.getUncommittedSizeInBytes(), equalTo(300L));
             assertThat(stats.getEarliestLastModifiedAge(), greaterThan(1L));
         }
 
-        final long expectedSizeInBytes = 358L;
+        final long expectedSizeInBytes = 355L;
         translog.rollGeneration();
         {
             final TranslogStats stats = stats();
@@ -725,14 +725,12 @@ public class TranslogTests extends ESTestCase {
                         assertEquals(expIndexOp.type(), indexOp.type());
                         assertEquals(expIndexOp.source(), indexOp.source());
                         assertEquals(expIndexOp.version(), indexOp.version());
-                        assertEquals(expIndexOp.versionType(), indexOp.versionType());
                         break;
                     case DELETE:
                         Translog.Delete delOp = (Translog.Delete) op;
                         Translog.Delete expDelOp = (Translog.Delete) expectedOp;
                         assertEquals(expDelOp.uid(), delOp.uid());
                         assertEquals(expDelOp.version(), delOp.version());
-                        assertEquals(expDelOp.versionType(), delOp.versionType());
                         break;
                     case NO_OP:
                         final Translog.NoOp noOp = (Translog.NoOp) op;
@@ -1478,7 +1476,7 @@ public class TranslogTests extends ESTestCase {
         try (Translog ignored = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
             fail("corrupted");
         } catch (IllegalStateException ex) {
-            assertEquals("Checkpoint file translog-3.ckp already exists but has corrupted content expected: Checkpoint{offset=3080, " +
+            assertEquals("Checkpoint file translog-3.ckp already exists but has corrupted content expected: Checkpoint{offset=3025, " +
                 "numOps=55, generation=3, minSeqNo=45, maxSeqNo=99, globalCheckpoint=-1, minTranslogGeneration=1, trimmedAboveSeqNo=-2} but got: Checkpoint{offset=0, numOps=0, " +
                 "generation=0, minSeqNo=-1, maxSeqNo=-1, globalCheckpoint=-1, minTranslogGeneration=0, trimmedAboveSeqNo=-2}", ex.getMessage());
         }
@@ -1842,8 +1840,7 @@ public class TranslogTests extends ESTestCase {
                                 new Term("_uid", threadId + "_" + opCount),
                                 seqNoGenerator.getAndIncrement(),
                                 primaryTerm.get(),
-                                1 + randomInt(100000),
-                                randomFrom(VersionType.values()));
+                                1 + randomInt(100000));
                             break;
                         case NO_OP:
                             op = new Translog.NoOp(seqNoGenerator.getAndIncrement(), primaryTerm.get(), randomAlphaOfLength(16));
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
index 537409f35d1..e7606328c76 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
@@ -122,22 +122,22 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase {
             final String indexName = orgReplica.shardId().getIndexName();
 
             // delete #1
-            orgReplica.applyDeleteOperationOnReplica(1, 2, "type", "id", VersionType.EXTERNAL);
+            orgReplica.applyDeleteOperationOnReplica(1, 2, "type", "id");
             getTranslog(orgReplica).rollGeneration(); // isolate the delete in it's own generation
             // index #0
-            orgReplica.applyIndexOperationOnReplica(0, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+            orgReplica.applyIndexOperationOnReplica(0, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
                 SourceToParse.source(indexName, "type", "id", new BytesArray("{}"), XContentType.JSON));
             // index #3
-            orgReplica.applyIndexOperationOnReplica(3, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+            orgReplica.applyIndexOperationOnReplica(3, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
                 SourceToParse.source(indexName, "type", "id-3", new BytesArray("{}"), XContentType.JSON));
             // Flushing a new commit with local checkpoint=1 allows to delete the translog gen #1.
             orgReplica.flush(new FlushRequest().force(true).waitIfOngoing(true));
             // index #2
-            orgReplica.applyIndexOperationOnReplica(2, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+            orgReplica.applyIndexOperationOnReplica(2, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
                 SourceToParse.source(indexName, "type", "id-2", new BytesArray("{}"), XContentType.JSON));
             orgReplica.updateGlobalCheckpointOnReplica(3L, "test");
             // index #5 -> force NoOp #4.
-            orgReplica.applyIndexOperationOnReplica(5, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+            orgReplica.applyIndexOperationOnReplica(5, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
                 SourceToParse.source(indexName, "type", "id-5", new BytesArray("{}"), XContentType.JSON));
 
             final int translogOps;
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index a23e29b0bcd..f2652224549 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -52,7 +52,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.Mapping;
@@ -493,14 +492,12 @@ public abstract class EngineTestCase extends ESTestCase {
 
     protected Engine.Index replicaIndexForDoc(ParsedDocument doc, long version, long seqNo,
                                             boolean isRetry) {
-        return new Engine.Index(newUid(doc), doc, seqNo, primaryTerm.get(), version, VersionType.EXTERNAL,
-                Engine.Operation.Origin.REPLICA, System.nanoTime(),
-                IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, isRetry);
+        return new Engine.Index(newUid(doc), doc, seqNo, primaryTerm.get(), version, null, Engine.Operation.Origin.REPLICA,
+            System.nanoTime(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, isRetry);
     }
 
     protected Engine.Delete replicaDeleteForDoc(String id, long version, long seqNo, long startTime) {
-        return new Engine.Delete("test", id, newUid(id), seqNo, 1, version, VersionType.EXTERNAL,
-            Engine.Operation.Origin.REPLICA, startTime);
+        return new Engine.Delete("test", id, newUid(id), seqNo, 1, version, null, Engine.Operation.Origin.REPLICA, startTime);
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
index 53fe89ac17e..9999a3b3748 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
@@ -124,14 +124,12 @@ public class TranslogHandler implements EngineConfig.TranslogRecoveryRunner {
                         source(indexName, index.type(), index.id(), index.source(),
                             XContentHelper.xContentType(index.source()))
                                 .routing(index.routing()), index.seqNo(), index.primaryTerm(),
-                        index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin,
-                        index.getAutoGeneratedIdTimestamp(), true);
+                        index.version(), null, origin, index.getAutoGeneratedIdTimestamp(), true);
                 return engineIndex;
             case DELETE:
                 final Translog.Delete delete = (Translog.Delete) operation;
                 final Engine.Delete engineDelete = new Engine.Delete(delete.type(), delete.id(), delete.uid(), delete.seqNo(),
-                        delete.primaryTerm(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(),
-                        origin, System.nanoTime());
+                        delete.primaryTerm(), delete.version(), null, origin, System.nanoTime());
                 return engineDelete;
             case NO_OP:
                 final Translog.NoOp noOp = (Translog.NoOp) operation;
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index 5a8e91841c5..e4849be20e1 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -573,7 +573,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
                 shard.getLocalCheckpoint());
         } else {
             result = shard.applyIndexOperationOnReplica(shard.seqNoStats().getMaxSeqNo() + 1, 0,
-                VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, sourceToParse);
+                IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, sourceToParse);
             if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
                 throw new TransportReplicationAction.RetryOnReplicaException(shard.shardId,
                     "Mappings are not available on the replica yet, triggered update: " + result.getRequiredMappingUpdate());
@@ -591,7 +591,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
         if (shard.routingEntry().primary()) {
             return shard.applyDeleteOperationOnPrimary(Versions.MATCH_ANY, type, id, VersionType.INTERNAL);
         } else {
-            return shard.applyDeleteOperationOnReplica(shard.seqNoStats().getMaxSeqNo() + 1, 0L, type, id, VersionType.EXTERNAL);
+            return shard.applyDeleteOperationOnReplica(shard.seqNoStats().getMaxSeqNo() + 1, 0L, type, id);
         }
     }
 

From 93d7468f3a49fba26a4176476b8f219e86ecdef0 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Tue, 17 Jul 2018 18:06:16 +0200
Subject: [PATCH 075/260] ESIndexLevelReplicationTestCase doesn't support
 replicated failures but it's good to know what they are

Sometimes we have a test failure that hits an `UnsupportedOperationException` in this infrastructure. When
debugging you want to know what caused this unexpected failure, but right now we're silent about it. This
commit adds some information to the `UnsupportedOperationException`

Relates to #32127
---
 .../index/replication/ESIndexLevelReplicationTestCase.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index ab18e359458..9de88216822 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -477,7 +477,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
 
             @Override
             public void failShard(String message, Exception exception) {
-                throw new UnsupportedOperationException();
+                throw new UnsupportedOperationException("failing a primary isn't supported. failure: " + message, exception);
             }
 
             @Override
@@ -550,13 +550,13 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
             public void failShardIfNeeded(ShardRouting replica, String message, Exception exception,
                                           Runnable onSuccess, Consumer<Exception> onPrimaryDemoted,
                                           Consumer<Exception> onIgnoredFailure) {
-                throw new UnsupportedOperationException();
+                throw new UnsupportedOperationException("failing shard " + replica + " isn't supported. failure: " + message, exception);
             }
 
             @Override
             public void markShardCopyAsStaleIfNeeded(ShardId shardId, String allocationId, Runnable onSuccess,
                                                      Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
-                throw new UnsupportedOperationException();
+                throw new UnsupportedOperationException("can't mark " + shardId  + ", aid [" + allocationId + "] as stale");
             }
         }
 

From 487cfc3b3f9df22f46e112461429578732844903 Mon Sep 17 00:00:00 2001
From: Toby McLaughlin <toby@jarpy.net>
Date: Wed, 18 Jul 2018 17:07:31 +1000
Subject: [PATCH 076/260] [DOCS] Update TLS on Docker for 6.3 (#32114)

Remove references to the `platinum` image and add a self-generated trial
licence to the example for TLS on Docker.

Fixes elastic/elasticsearch-docker#176
---
 .../configuring-tls-docker.asciidoc           | 29 ++++++++++++++-----
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/x-pack/docs/en/security/securing-communications/configuring-tls-docker.asciidoc b/x-pack/docs/en/security/securing-communications/configuring-tls-docker.asciidoc
index d93d4e523d9..49913382482 100644
--- a/x-pack/docs/en/security/securing-communications/configuring-tls-docker.asciidoc
+++ b/x-pack/docs/en/security/securing-communications/configuring-tls-docker.asciidoc
@@ -1,12 +1,13 @@
 [role="xpack"]
 [[configuring-tls-docker]]
-=== Encrypting Communications in an {es} Docker Image
+=== Encrypting Communications in an {es} Docker Container
 
 Starting with version 6.0.0, {security} (Gold, Platinum or Enterprise subscriptions) https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking-6.0.0-xes.html[requires SSL/TLS]
 encryption for the transport networking layer.
 
 This section demonstrates an easy path to get started with SSL/TLS for both
-HTTPS and transport using the `elasticsearch-platinum` docker image.
+HTTPS and transport using the {es} Docker image. The example uses
+Docker Compose to manage the containers.
 
 For further details, please refer to
 {xpack-ref}/encrypting-communications.html[Encrypting Communications] and
@@ -17,7 +18,7 @@ https://www.elastic.co/subscriptions[available subscriptions].
 
 <<docker,Install {es} with Docker>>.
 
-Inside a new, empty, directory create the following **four files**:
+Inside a new, empty directory, create the following four files:
 
 `instances.yml`:
 ["source","yaml"]
@@ -29,6 +30,7 @@ instances:
       - localhost
     ip:
       - 127.0.0.1
+
   - name: es02
     dns:
       - es02
@@ -60,6 +62,7 @@ ifeval::["{release-state}"!="unreleased"]
 ["source","yaml",subs="attributes"]
 ----
 version: '2.2'
+
 services:
   create_certs:
     container_name: create_certs
@@ -96,6 +99,7 @@ ifeval::["{release-state}"!="unreleased"]
 ["source","yaml",subs="attributes"]
 ----
 version: '2.2'
+
 services:
   es01:
     container_name: es01
@@ -105,9 +109,11 @@ services:
       - discovery.zen.minimum_master_nodes=2
       - ELASTIC_PASSWORD=$ELASTIC_PASSWORD <1>
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - xpack.license.self_generated.type=trial <2>
+      - xpack.security.enabled=true
       - xpack.security.http.ssl.enabled=true
       - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.verification_mode=certificate <2>
+      - xpack.security.transport.ssl.verification_mode=certificate <3>
       - xpack.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
       - xpack.ssl.certificate=$CERTS_DIR/es01/es01.crt
       - xpack.ssl.key=$CERTS_DIR/es01/es01.key
@@ -119,15 +125,18 @@ services:
       interval: 30s
       timeout: 10s
       retries: 5
+
   es02:
     container_name: es02
-    image: docker.elastic.co/elasticsearch/elasticsearch-platinum:{version}
+    image: docker.elastic.co/elasticsearch/elasticsearch:{version}
     environment:
       - node.name=es02
       - discovery.zen.minimum_master_nodes=2
       - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
       - discovery.zen.ping.unicast.hosts=es01
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - xpack.license.self_generated.type=trial
+      - xpack.security.enabled=true
       - xpack.security.http.ssl.enabled=true
       - xpack.security.transport.ssl.enabled=true
       - xpack.security.transport.ssl.verification_mode=certificate
@@ -135,16 +144,20 @@ services:
       - xpack.ssl.certificate=$CERTS_DIR/es02/es02.crt
       - xpack.ssl.key=$CERTS_DIR/es02/es02.key
     volumes: ['esdata_02:/usr/share/elasticsearch/data', './certs:$CERTS_DIR']
+
   wait_until_ready:
-    image: docker.elastic.co/elasticsearch/elasticsearch-platinum:{version}
+    image: docker.elastic.co/elasticsearch/elasticsearch:{version}
     command: /usr/bin/true
     depends_on: {"es01": {"condition": "service_healthy"}}
+
 volumes: {"esdata_01": {"driver": "local"}, "esdata_02": {"driver": "local"}}
 ----
 
-<1> Bootstrap `elastic` with the password defined in `.env`. See 
+<1> Bootstrap `elastic` with the password defined in `.env`. See
 {stack-ov}/built-in-users.html#bootstrap-elastic-passwords[the Elastic Bootstrap Password].
-<2> Disable verification of authenticity for inter-node communication. Allows
+<2> Automatically generate and apply a trial subscription, in order to enable
+{security}.
+<3> Disable verification of authenticity for inter-node communication. Allows
 creating self-signed certificates without having to pin specific internal IP addresses.
 endif::[]
 

From 15f95a9f934d7ff4a4c3ab9e70a8877cf046e9ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Wed, 18 Jul 2018 09:12:28 +0200
Subject: [PATCH 077/260] Fix `range` queries on `_type` field for singe type
 indices (#31756)

With the introduction of single types in 6.x, the `_type` field is no longer
indexed, which leads to certain queries that were working before throw errors
now. One such query is the `range` query, that, if performed on a single typer
index, currently throws an IAE since the field is not indexed.
This change adds special treatment for this case in the TypeFieldMapper,
comparing the range queries lower and upper bound to the one existing type and
either returns a MatchAllDocs or a MatchNoDocs query.

Relates to #31632
Closes #31476
---
 .../index/mapper/TypeFieldMapper.java         | 27 ++++++++++++++
 .../search/query/SearchQueryIT.java           | 36 +++++++++++++++++++
 2 files changed, 63 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
index ffb548fd0f1..71bd2e93d30 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
@@ -35,6 +35,8 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -90,6 +92,8 @@ public class TypeFieldMapper extends MetadataFieldMapper {
 
     static final class TypeFieldType extends StringFieldType {
 
+        private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(ESLoggerFactory.getLogger(TypeFieldType.class));
+
         TypeFieldType() {
         }
 
@@ -154,6 +158,29 @@ public class TypeFieldMapper extends MetadataFieldMapper {
             }
         }
 
+        @Override
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
+            DEPRECATION_LOGGER.deprecatedAndMaybeLog("range_single_type",
+                    "Running [range] query on [_type] field for an index with a single type. As types are deprecated, this functionality will be removed in future releases.");
+            Query result = new MatchAllDocsQuery();
+            String type = context.getMapperService().documentMapper().type();
+            if (type != null) {
+                BytesRef typeBytes = new BytesRef(type);
+                if (lowerTerm != null) {
+                    int comp = indexedValueForSearch(lowerTerm).compareTo(typeBytes);
+                    if (comp > 0 || (comp == 0 && includeLower == false)) {
+                        result = new MatchNoDocsQuery("[_type] was lexicographically smaller than lower bound of range");
+                    }
+                }
+                if (upperTerm != null) {
+                    int comp = indexedValueForSearch(upperTerm).compareTo(typeBytes);
+                    if (comp < 0 || (comp == 0 && includeUpper == false)) {
+                        result = new MatchNoDocsQuery("[_type] was lexicographically greater than upper bound of range");
+                    }
+                }
+            }
+            return result;
+        }
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index be71867edd2..1694f86c53e 100644
--- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -1822,4 +1822,40 @@ public class SearchQueryIT extends ESIntegTestCase {
         SearchResponse searchResponse = client().prepareSearch("test").setQuery(range).get();
         assertHitCount(searchResponse, 1);
     }
+
+    public void testRangeQueryTypeField_31476() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("foo", "field", "type=keyword"));
+
+        client().prepareIndex("test", "foo", "1").setSource("field", "value").get();
+        refresh();
+
+        RangeQueryBuilder range = new RangeQueryBuilder("_type").from("ape").to("zebra");
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(range).get();
+        assertHitCount(searchResponse, 1);
+
+        range = new RangeQueryBuilder("_type").from("monkey").to("zebra");
+        searchResponse = client().prepareSearch("test").setQuery(range).get();
+        assertHitCount(searchResponse, 0);
+
+        range = new RangeQueryBuilder("_type").from("ape").to("donkey");
+        searchResponse = client().prepareSearch("test").setQuery(range).get();
+        assertHitCount(searchResponse, 0);
+
+        range = new RangeQueryBuilder("_type").from("ape").to("foo").includeUpper(false);
+        searchResponse = client().prepareSearch("test").setQuery(range).get();
+        assertHitCount(searchResponse, 0);
+
+        range = new RangeQueryBuilder("_type").from("ape").to("foo").includeUpper(true);
+        searchResponse = client().prepareSearch("test").setQuery(range).get();
+        assertHitCount(searchResponse, 1);
+
+        range = new RangeQueryBuilder("_type").from("foo").to("zebra").includeLower(false);
+        searchResponse = client().prepareSearch("test").setQuery(range).get();
+        assertHitCount(searchResponse, 0);
+
+        range = new RangeQueryBuilder("_type").from("foo").to("zebra").includeLower(true);
+        searchResponse = client().prepareSearch("test").setQuery(range).get();
+        assertHitCount(searchResponse, 1);
+    }
+
 }

From 69cbdc47eb97544dfde51a042860fca110c6a20c Mon Sep 17 00:00:00 2001
From: Alpar Torok <torokalpar@gmail.com>
Date: Wed, 18 Jul 2018 08:23:59 +0000
Subject: [PATCH 078/260] Fix CP for namingConventions when gradle home has
 spaces (#31914)

* Fix CP for namingConventions when gradle home has spaces

Closes #31736.

Probably not Windows specific, just not common to have spaces on Linux.
---
 .../precommit/NamingConventionsTask.java      | 35 +++++++++++--------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/NamingConventionsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/NamingConventionsTask.java
index cfbb75456bc..297586e9ac6 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/NamingConventionsTask.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/NamingConventionsTask.java
@@ -16,6 +16,8 @@ import org.gradle.api.tasks.SourceSetContainer;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.util.Objects;
 
 /**
@@ -30,16 +32,25 @@ public class NamingConventionsTask extends LoggedExec {
         final Project project = getProject();
 
         SourceSetContainer sourceSets = getJavaSourceSets();
-        final FileCollection classpath = project.files(
-            // This works because the class only depends on one class from junit that will be available from the
-            // tests compile classpath. It's the most straight forward way of telling Java where to find the main
-            // class.
-            NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation().getPath(),
-            // the tests to be loaded
-            checkForTestsInMain ? sourceSets.getByName("main").getRuntimeClasspath() : project.files(),
-            sourceSets.getByName("test").getCompileClasspath(),
-            sourceSets.getByName("test").getOutput()
-        );
+        final FileCollection classpath;
+        try {
+            URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation();
+            if (location.getProtocol().equals("file") == false) {
+                throw new GradleException("Unexpected location for NamingConventionCheck class: "+ location);
+            }
+            classpath = project.files(
+                    // This works because the class only depends on one class from junit that will be available from the
+                    // tests compile classpath. It's the most straight forward way of telling Java where to find the main
+                    // class.
+                    location.toURI().getPath(),
+                    // the tests to be loaded
+                    checkForTestsInMain ? sourceSets.getByName("main").getRuntimeClasspath() : project.files(),
+                    sourceSets.getByName("test").getCompileClasspath(),
+                    sourceSets.getByName("test").getOutput()
+            );
+        } catch (URISyntaxException e) {
+            throw new AssertionError(e);
+        }
         dependsOn(project.getTasks().matching(it -> "testCompileClasspath".equals(it.getName())));
         getInputs().files(classpath);
 
@@ -111,10 +122,6 @@ public class NamingConventionsTask extends LoggedExec {
         this.successMarker = successMarker;
     }
 
-    public boolean getSkipIntegTestInDisguise() {
-        return skipIntegTestInDisguise;
-    }
-
     public boolean isSkipIntegTestInDisguise() {
         return skipIntegTestInDisguise;
     }

From ef5e8d8d8a82afcc9d9f49a4c8a868d559ffef93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Wed, 18 Jul 2018 10:34:42 +0200
Subject: [PATCH 079/260] Fix Java 11 javadoc compile problem

Java 11 complains with a "type arguments not allowed here" error when types are
used in javadoc links it seems. Simply removing it.
---
 .../java/org/elasticsearch/painless/PainlessDocGenerator.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index 4486a52ccb1..cc596dcc395 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -328,7 +328,7 @@ public class PainlessDocGenerator {
     }
 
     /**
-     * Pick the javadoc root for a {@link Class<?>}.
+     * Pick the javadoc root for a {@link Class}.
      */
     private static String javadocRoot(Class<?> clazz) {
         String classPackage = clazz.getPackage().getName();

From 5856c396ddd9e36094c5471d436be100bb586e60 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Wed, 18 Jul 2018 11:30:44 +0200
Subject: [PATCH 080/260] A replica can be promoted and started in one cluster
 state update (#32042)

When a replica is fully recovered (i.e., in `POST_RECOVERY` state) we send a request to the master
to start the shard. The master changes the state of the replica and publishes a cluster state to that
effect. In certain cases, that cluster state can be processed on the node hosting the replica
*together* with a cluster state that promotes that, now started, replica to a primary. This can
happen due to cluster state batched processing or if the master died after having committed the
cluster state that starts the shard but before publishing it to the node with the replica. If the master
also held the primary shard, the new master node will remove the primary (as it failed) and will also
immediately promote the replica (thinking it is started).

Sadly our code in IndexShard didn't allow for this which caused [assertions](https://github.com/elastic/elasticsearch/blob/13917162ad5c59a96ccb4d6a81a5044546c45c22/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java#L482) to be tripped in some of our tests runs.
---
 .../elasticsearch/index/shard/IndexShard.java | 19 ++--
 .../IndexLevelReplicationTests.java           |  2 +-
 .../index/shard/IndexShardTests.java          | 67 +++++---------
 ...actIndicesClusterStateServiceTestCase.java |  8 ++
 .../PeerRecoveryTargetServiceTests.java       |  2 +-
 .../indices/recovery/RecoveryTests.java       |  2 +-
 .../ESIndexLevelReplicationTestCase.java      |  9 +-
 .../index/shard/IndexShardTestCase.java       | 90 +++++++++++++++----
 8 files changed, 126 insertions(+), 73 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index b07e22875e8..fc08438a7d9 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -413,10 +413,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
 
             if (state == IndexShardState.POST_RECOVERY && newRouting.active()) {
                 assert currentRouting.active() == false : "we are in POST_RECOVERY, but our shard routing is active " + currentRouting;
-
-                if (newRouting.primary() && currentRouting.isRelocationTarget() == false) {
-                    replicationTracker.activatePrimaryMode(getLocalCheckpoint());
-                }
+                assert currentRouting.isRelocationTarget()  == false || currentRouting.primary() == false ||
+                        replicationTracker.isPrimaryMode() :
+                    "a primary relocation is completed by the master, but primary mode is not active " + currentRouting;
 
                 changeState(IndexShardState.STARTED, "global state is [" + newRouting.state() + "]");
             } else if (currentRouting.primary() && currentRouting.relocating() && replicationTracker.isPrimaryMode() == false &&
@@ -432,7 +431,12 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             final CountDownLatch shardStateUpdated = new CountDownLatch(1);
 
             if (newRouting.primary()) {
-                if (newPrimaryTerm != primaryTerm) {
+                if (newPrimaryTerm == primaryTerm) {
+                    if (currentRouting.initializing() && currentRouting.isRelocationTarget() == false && newRouting.active()) {
+                        // the master started a recovering primary, activate primary mode.
+                        replicationTracker.activatePrimaryMode(getLocalCheckpoint());
+                    }
+                } else {
                     assert currentRouting.primary() == false : "term is only increased as part of primary promotion";
                     /* Note that due to cluster state batching an initializing primary shard term can failed and re-assigned
                      * in one state causing it's term to be incremented. Note that if both current shard state and new
@@ -521,6 +525,11 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             }
             // set this last, once we finished updating all internal state.
             this.shardRouting = newRouting;
+
+            assert this.shardRouting.primary() == false ||
+                this.shardRouting.started() == false || // note that we use started and not active to avoid relocating shards
+                this.replicationTracker.isPrimaryMode()
+                : "an started primary must be in primary mode " + this.shardRouting;
             shardStateUpdated.countDown();
         }
         if (currentRouting != null && currentRouting.active() == false && newRouting.active()) {
diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
index 018548be962..b05b1e5cc5c 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -363,7 +363,7 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase
             logger.info("--> Promote replica2 as the primary");
             shards.promoteReplicaToPrimary(replica2);
             logger.info("--> Recover replica3 from replica2");
-            recoverReplica(replica3, replica2);
+            recoverReplica(replica3, replica2, true);
             try (Translog.Snapshot snapshot = getTranslog(replica3).newSnapshot()) {
                 assertThat(snapshot.totalOperations(), equalTo(initDocs + 1));
                 assertThat(snapshot.next(), equalTo(op2));
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 2e07ec950a5..15e6151457f 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -226,6 +226,7 @@ public class IndexShardTests extends IndexShardTestCase {
     }
 
     public void testFailShard() throws Exception {
+        allowShardFailures();
         IndexShard shard = newStartedShard();
         final ShardPath shardPath = shard.shardPath();
         assertNotNull(shardPath);
@@ -309,7 +310,8 @@ public class IndexShardTests extends IndexShardTestCase {
     }
 
     public void testPrimaryPromotionDelaysOperations() throws IOException, BrokenBarrierException, InterruptedException {
-        final IndexShard indexShard = newStartedShard(false);
+        final IndexShard indexShard = newShard(false);
+        recoveryEmptyReplica(indexShard, randomBoolean());
 
         final int operations = scaledRandomIntBetween(1, 64);
         final CyclicBarrier barrier = new CyclicBarrier(1 + operations);
@@ -353,20 +355,10 @@ public class IndexShardTests extends IndexShardTestCase {
         barrier.await();
         latch.await();
 
-        // promote the replica
         final ShardRouting replicaRouting = indexShard.routingEntry();
-        final ShardRouting primaryRouting =
-                newShardRouting(
-                        replicaRouting.shardId(),
-                        replicaRouting.currentNodeId(),
-                        null,
-                        true,
-                        ShardRoutingState.STARTED,
-                        replicaRouting.allocationId());
-        indexShard.updateShardState(primaryRouting, indexShard.getPrimaryTerm() + 1, (shard, listener) -> {},
-            0L, Collections.singleton(primaryRouting.allocationId().getId()),
-            new IndexShardRoutingTable.Builder(primaryRouting.shardId()).addShard(primaryRouting).build(),
-            Collections.emptySet());
+        promoteReplica(indexShard, Collections.singleton(replicaRouting.allocationId().getId()),
+            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(replicaRouting).build());
+
 
         final int delayedOperations = scaledRandomIntBetween(1, 64);
         final CyclicBarrier delayedOperationsBarrier = new CyclicBarrier(1 + delayedOperations);
@@ -428,8 +420,9 @@ public class IndexShardTests extends IndexShardTestCase {
      * 1) Internal state (ala ReplicationTracker) have been updated
      * 2) Primary term is set to the new term
      */
-    public void testPublishingOrderOnPromotion() throws IOException, BrokenBarrierException, InterruptedException {
-        final IndexShard indexShard = newStartedShard(false);
+    public void testPublishingOrderOnPromotion() throws IOException, InterruptedException, BrokenBarrierException {
+        final IndexShard indexShard = newShard(false);
+        recoveryEmptyReplica(indexShard, randomBoolean());
         final long promotedTerm = indexShard.getPrimaryTerm() + 1;
         final CyclicBarrier barrier = new CyclicBarrier(2);
         final AtomicBoolean stop = new AtomicBoolean();
@@ -448,18 +441,10 @@ public class IndexShardTests extends IndexShardTestCase {
         });
         thread.start();
 
-        final ShardRouting replicaRouting = indexShard.routingEntry();
-        final ShardRouting primaryRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), null, true,
-            ShardRoutingState.STARTED, replicaRouting.allocationId());
-
-
-        final Set<String> inSyncAllocationIds = Collections.singleton(primaryRouting.allocationId().getId());
-        final IndexShardRoutingTable routingTable =
-            new IndexShardRoutingTable.Builder(primaryRouting.shardId()).addShard(primaryRouting).build();
         barrier.await();
-        // promote the replica
-        indexShard.updateShardState(primaryRouting, promotedTerm, (shard, listener) -> {}, 0L, inSyncAllocationIds, routingTable,
-            Collections.emptySet());
+        final ShardRouting replicaRouting = indexShard.routingEntry();
+        promoteReplica(indexShard, Collections.singleton(replicaRouting.allocationId().getId()),
+            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(replicaRouting).build());
 
         stop.set(true);
         thread.join();
@@ -468,7 +453,8 @@ public class IndexShardTests extends IndexShardTestCase {
 
 
     public void testPrimaryFillsSeqNoGapsOnPromotion() throws Exception {
-        final IndexShard indexShard = newStartedShard(false);
+        final IndexShard indexShard = newShard(false);
+        recoveryEmptyReplica(indexShard, randomBoolean());
 
         // most of the time this is large enough that most of the time there will be at least one gap
         final int operations = 1024 - scaledRandomIntBetween(0, 1024);
@@ -479,17 +465,8 @@ public class IndexShardTests extends IndexShardTestCase {
 
         // promote the replica
         final ShardRouting replicaRouting = indexShard.routingEntry();
-        final ShardRouting primaryRouting =
-                newShardRouting(
-                        replicaRouting.shardId(),
-                        replicaRouting.currentNodeId(),
-                        null,
-                        true,
-                        ShardRoutingState.STARTED,
-                        replicaRouting.allocationId());
-        indexShard.updateShardState(primaryRouting, indexShard.getPrimaryTerm() + 1, (shard, listener) -> {},
-            0L, Collections.singleton(primaryRouting.allocationId().getId()),
-            new IndexShardRoutingTable.Builder(primaryRouting.shardId()).addShard(primaryRouting).build(), Collections.emptySet());
+        promoteReplica(indexShard, Collections.singleton(replicaRouting.allocationId().getId()),
+            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(replicaRouting).build());
 
         /*
          * This operation completing means that the delay operation executed as part of increasing the primary term has completed and the
@@ -506,7 +483,7 @@ public class IndexShardTests extends IndexShardTestCase {
 
                     @Override
                     public void onFailure(Exception e) {
-                        throw new RuntimeException(e);
+                        throw new AssertionError(e);
                     }
                 },
                 ThreadPool.Names.GENERIC, "");
@@ -846,7 +823,7 @@ public class IndexShardTests extends IndexShardTestCase {
         // add a replica
         recoverShardFromStore(primaryShard);
         final IndexShard replicaShard = newShard(shardId, false);
-        recoverReplica(replicaShard, primaryShard);
+        recoverReplica(replicaShard, primaryShard, true);
         final int maxSeqNo = randomIntBetween(0, 128);
         for (int i = 0; i <= maxSeqNo; i++) {
             EngineTestCase.generateNewSeqNo(primaryShard.getEngine());
@@ -1625,7 +1602,7 @@ public class IndexShardTests extends IndexShardTestCase {
         IndexShardTestCase.updateRoutingEntry(primarySource, primarySource.routingEntry().relocate(randomAlphaOfLength(10), -1));
         final IndexShard primaryTarget = newShard(primarySource.routingEntry().getTargetRelocatingShard());
         updateMappings(primaryTarget, primarySource.indexSettings().getIndexMetaData());
-        recoverReplica(primaryTarget, primarySource);
+        recoverReplica(primaryTarget, primarySource, true);
 
         // check that local checkpoint of new primary is properly tracked after primary relocation
         assertThat(primaryTarget.getLocalCheckpoint(), equalTo(totalOps - 1L));
@@ -2082,7 +2059,7 @@ public class IndexShardTests extends IndexShardTestCase {
                     assertFalse(replica.isSyncNeeded());
                     return localCheckpoint;
                 }
-            }, true);
+            }, true, true);
 
         closeShards(primary, replica);
     }
@@ -2189,7 +2166,7 @@ public class IndexShardTests extends IndexShardTestCase {
                     assertTrue(replica.isActive());
                     return localCheckpoint;
                 }
-            }, false);
+            }, false, true);
 
         closeShards(primary, replica);
     }
@@ -2241,7 +2218,7 @@ public class IndexShardTests extends IndexShardTestCase {
                     super.finalizeRecovery(globalCheckpoint);
                     assertListenerCalled.accept(replica);
                 }
-            }, false);
+            }, false, true);
 
         closeShards(primary, replica);
     }
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
index 35bbc497838..5c6b000f7e5 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
@@ -357,6 +357,14 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC
                 assertTrue("and active shard must stay active, current: " + this.shardRouting + ", got: " + shardRouting,
                     shardRouting.active());
             }
+            if (this.shardRouting.primary()) {
+                assertTrue("a primary shard can't be demoted", shardRouting.primary());
+            } else if (shardRouting.primary()) {
+                // note: it's ok for a replica in post recovery to be started and promoted at once
+                // this can happen when the primary failed after we sent the start shard message
+                assertTrue("a replica can only be promoted when active. current: " + this.shardRouting + " new: " + shardRouting,
+                    shardRouting.active());
+            }
             this.shardRouting = shardRouting;
             if (shardRouting.primary()) {
                 term = newPrimaryTerm;
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java
index 3b50fa64915..4b1419375e6 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java
@@ -43,7 +43,7 @@ public class PeerRecoveryTargetServiceTests extends IndexShardTestCase {
         try {
             // Empty store
             {
-                recoveryEmptyReplica(replica);
+                recoveryEmptyReplica(replica, true);
                 final RecoveryTarget recoveryTarget = new RecoveryTarget(replica, null, null, null);
                 assertThat(PeerRecoveryTargetService.getStartingSeqNo(logger, recoveryTarget), equalTo(0L));
                 recoveryTarget.decRef();
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
index e7606328c76..aaba17c3151 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
@@ -261,7 +261,7 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase {
         }
         IndexShard replicaShard = newShard(primaryShard.shardId(), false);
         updateMappings(replicaShard, primaryShard.indexSettings().getIndexMetaData());
-        recoverReplica(replicaShard, primaryShard);
+        recoverReplica(replicaShard, primaryShard, true);
         List<IndexCommit> commits = DirectoryReader.listCommits(replicaShard.store().directory());
         long maxSeqNo = Long.parseLong(commits.get(0).getUserData().get(SequenceNumbers.MAX_SEQ_NO));
         assertThat(maxSeqNo, lessThanOrEqualTo(globalCheckpoint));
diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 9de88216822..5a5ee12065c 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -265,7 +265,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
                 RecoverySource.PeerRecoverySource.INSTANCE);
 
             final IndexShard newReplica =
-                    newShard(shardRouting, shardPath, indexMetaData, null, getEngineFactory(shardRouting), () -> {}, EMPTY_EVENT_LISTENER);
+                newShard(shardRouting, shardPath, indexMetaData, null, getEngineFactory(shardRouting), () -> {}, EMPTY_EVENT_LISTENER);
             replicas.add(newReplica);
             updateAllocationIDsOnPrimary();
             return newReplica;
@@ -341,8 +341,11 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
             IndexShard replica,
             BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
             boolean markAsRecovering) throws IOException {
-            ESIndexLevelReplicationTestCase.this.recoverReplica(replica, primary, targetSupplier, markAsRecovering, activeIds(),
-                routingTable(Function.identity()));
+            final IndexShardRoutingTable routingTable = routingTable(Function.identity());
+            final Set<String> inSyncIds = activeIds();
+            ESIndexLevelReplicationTestCase.this.recoverUnstartedReplica(replica, primary, targetSupplier, markAsRecovering, inSyncIds,
+                routingTable);
+            ESIndexLevelReplicationTestCase.this.startReplicaAfterRecovery(replica, primary, inSyncIds, routingTable);
         }
 
         public synchronized DiscoveryNode getPrimaryNode() {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index e4849be20e1..0cbc6e44502 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -92,8 +92,10 @@ import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BiFunction;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting;
 import static org.hamcrest.Matchers.contains;
@@ -108,6 +110,14 @@ public abstract class IndexShardTestCase extends ESTestCase {
 
     public static final IndexEventListener EMPTY_EVENT_LISTENER = new IndexEventListener() {};
 
+    private static final AtomicBoolean failOnShardFailures = new AtomicBoolean(true);
+
+    private static final Consumer<IndexShard.ShardFailure> DEFAULT_SHARD_FAILURE_HANDLER = failure -> {
+        if (failOnShardFailures.get()) {
+            throw new AssertionError(failure.reason, failure.cause);
+        }
+    };
+
     protected static final PeerRecoveryTargetService.RecoveryListener recoveryListener = new PeerRecoveryTargetService.RecoveryListener() {
         @Override
         public void onRecoveryDone(RecoveryState state) {
@@ -128,6 +138,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
         super.setUp();
         threadPool = new TestThreadPool(getClass().getName(), threadPoolSettings());
         primaryTerm = randomIntBetween(1, 100); // use random but fixed term for creating shards
+        failOnShardFailures.set(true);
     }
 
     @Override
@@ -139,6 +150,15 @@ public abstract class IndexShardTestCase extends ESTestCase {
         }
     }
 
+    /**
+     * by default, tests will fail if any shard created by this class fails. Tests that cause failures by design
+     * can call this method to ignore those failures
+     *
+     */
+    protected void allowShardFailures() {
+        failOnShardFailures.set(false);
+    }
+
     public Settings threadPoolSettings() {
         return Settings.EMPTY;
     }
@@ -270,7 +290,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
 
     /**
      * creates a new initializing shard.
-     *  @param routing                shard routing to use
+     * @param routing                shard routing to use
      * @param shardPath              path to use for shard data
      * @param indexMetaData          indexMetaData for the shard, including any mapping
      * @param indexSearcherWrapper   an optional wrapper to be used during searchers
@@ -302,6 +322,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
                 engineFactory, indexEventListener, indexSearcherWrapper, threadPool,
                 BigArrays.NON_RECYCLING_INSTANCE, warmer, Collections.emptyList(), Arrays.asList(listeners), globalCheckpointSyncer,
                 breakerService);
+            indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER);
             success = true;
         } finally {
             if (success == false) {
@@ -358,7 +379,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
         if (primary) {
             recoverShardFromStore(shard);
         } else {
-            recoveryEmptyReplica(shard);
+            recoveryEmptyReplica(shard, true);
         }
         return shard;
     }
@@ -399,11 +420,11 @@ public abstract class IndexShardTestCase extends ESTestCase {
             inSyncIds, newRoutingTable, Collections.emptySet());
     }
 
-    protected void recoveryEmptyReplica(IndexShard replica) throws IOException {
+    protected void recoveryEmptyReplica(IndexShard replica, boolean startReplica) throws IOException {
         IndexShard primary = null;
         try {
             primary = newStartedShard(true);
-            recoverReplica(replica, primary);
+            recoverReplica(replica, primary, startReplica);
         } finally {
             closeShards(primary);
         }
@@ -415,42 +436,48 @@ public abstract class IndexShardTestCase extends ESTestCase {
     }
 
     /** recovers a replica from the given primary **/
-    protected void recoverReplica(IndexShard replica, IndexShard primary) throws IOException {
+    protected void recoverReplica(IndexShard replica, IndexShard primary, boolean startReplica) throws IOException {
         recoverReplica(replica, primary,
             (r, sourceNode) -> new RecoveryTarget(r, sourceNode, recoveryListener, version -> {
             }),
-            true);
+            true, true);
     }
 
     /** recovers a replica from the given primary **/
     protected void recoverReplica(final IndexShard replica,
                                   final IndexShard primary,
                                   final BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
-                                  final boolean markAsRecovering) throws IOException {
+                                  final boolean markAsRecovering, final boolean markAsStarted) throws IOException {
         IndexShardRoutingTable.Builder newRoutingTable = new IndexShardRoutingTable.Builder(replica.shardId());
         newRoutingTable.addShard(primary.routingEntry());
         if (replica.routingEntry().isRelocationTarget() == false) {
             newRoutingTable.addShard(replica.routingEntry());
         }
-        recoverReplica(replica, primary, targetSupplier, markAsRecovering,
-            Collections.singleton(primary.routingEntry().allocationId().getId()),
-            newRoutingTable.build());
+        final Set<String> inSyncIds = Collections.singleton(primary.routingEntry().allocationId().getId());
+        final IndexShardRoutingTable routingTable = newRoutingTable.build();
+        recoverUnstartedReplica(replica, primary, targetSupplier, markAsRecovering, inSyncIds, routingTable);
+        if (markAsStarted) {
+            startReplicaAfterRecovery(replica, primary, inSyncIds, routingTable);
+        }
     }
 
     /**
      * Recovers a replica from the give primary, allow the user to supply a custom recovery target. A typical usage of a custom recovery
      * target is to assert things in the various stages of recovery.
+     *
+     * Note: this method keeps the shard in {@link IndexShardState#POST_RECOVERY} and doesn't start it.
+     *
      * @param replica                the recovery target shard
      * @param primary                the recovery source shard
      * @param targetSupplier         supplies an instance of {@link RecoveryTarget}
      * @param markAsRecovering       set to {@code false} if the replica is marked as recovering
      */
-    protected final void recoverReplica(final IndexShard replica,
-                                        final IndexShard primary,
-                                        final BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
-                                        final boolean markAsRecovering,
-                                        final Set<String> inSyncIds,
-                                        final IndexShardRoutingTable routingTable) throws IOException {
+    protected final void recoverUnstartedReplica(final IndexShard replica,
+                                                 final IndexShard primary,
+                                                 final BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
+                                                 final boolean markAsRecovering,
+                                                 final Set<String> inSyncIds,
+                                                 final IndexShardRoutingTable routingTable) throws IOException {
         final DiscoveryNode pNode = getFakeDiscoNode(primary.routingEntry().currentNodeId());
         final DiscoveryNode rNode = getFakeDiscoNode(replica.routingEntry().currentNodeId());
         if (markAsRecovering) {
@@ -478,11 +505,15 @@ public abstract class IndexShardTestCase extends ESTestCase {
             request,
             (int) ByteSizeUnit.MB.toBytes(1),
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), pNode.getName()).build());
-        final ShardRouting initializingReplicaRouting = replica.routingEntry();
         primary.updateShardState(primary.routingEntry(), primary.getPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(),
             inSyncIds, routingTable, Collections.emptySet());
         recovery.recoverToTarget();
         recoveryTarget.markAsDone();
+    }
+
+    protected void startReplicaAfterRecovery(IndexShard replica, IndexShard primary, Set<String> inSyncIds,
+                                             IndexShardRoutingTable routingTable) throws IOException {
+        ShardRouting initializingReplicaRouting = replica.routingEntry();
         IndexShardRoutingTable newRoutingTable =
             initializingReplicaRouting.isRelocationTarget() ?
                 new IndexShardRoutingTable.Builder(routingTable)
@@ -502,6 +533,31 @@ public abstract class IndexShardTestCase extends ESTestCase {
             currentClusterStateVersion.get(), inSyncIdsWithReplica, newRoutingTable, Collections.emptySet());
     }
 
+
+    /**
+     * promotes a replica to primary, incrementing it's term and starting it if needed
+     */
+    protected void promoteReplica(IndexShard replica, Set<String> inSyncIds, IndexShardRoutingTable routingTable) throws IOException {
+        assertThat(inSyncIds, contains(replica.routingEntry().allocationId().getId()));
+        final ShardRouting routingEntry = newShardRouting(
+            replica.routingEntry().shardId(),
+            replica.routingEntry().currentNodeId(),
+            null,
+            true,
+            ShardRoutingState.STARTED,
+            replica.routingEntry().allocationId());
+
+        final IndexShardRoutingTable newRoutingTable = new IndexShardRoutingTable.Builder(routingTable)
+            .removeShard(replica.routingEntry())
+            .addShard(routingEntry)
+            .build();
+        replica.updateShardState(routingEntry, replica.getPrimaryTerm() + 1,
+            (is, listener) ->
+                listener.onResponse(new PrimaryReplicaSyncer.ResyncTask(1, "type", "action", "desc", null, Collections.emptyMap())),
+            currentClusterStateVersion.incrementAndGet(),
+            inSyncIds, newRoutingTable, Collections.emptySet());
+    }
+
     private Store.MetadataSnapshot getMetadataSnapshotOrEmpty(IndexShard replica) throws IOException {
         Store.MetadataSnapshot result;
         try {

From 8235b254abf274e99b7af91d125edc0a9dbe627a Mon Sep 17 00:00:00 2001
From: Vladimir Dolzhenko <vladimir.dolzhenko@gmail.com>
Date: Wed, 18 Jul 2018 12:18:00 +0200
Subject: [PATCH 081/260] Add EC2 credential test for repository-s3 (#31918)

Add EC2 credential test for repository-s3

Relates to #26913
---
 .../gradle/test/ClusterConfiguration.groovy   |   7 +-
 .../gradle/test/ClusterFormationTasks.groovy  |  29 ++-
 .../elasticsearch/gradle/test/NodeInfo.groovy |  10 +-
 plugins/repository-s3/build.gradle            |  50 +++-
 .../repositories/s3/AmazonS3Fixture.java      | 226 ++++++++++++----
 .../40_repository_ec2_credentials.yml         | 243 ++++++++++++++++++
 6 files changed, 485 insertions(+), 80 deletions(-)
 create mode 100644 plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index 5c363ac043a..d6477e05b15 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -137,7 +137,10 @@ class ClusterConfiguration {
         this.project = project
     }
 
-    Map<String, String> systemProperties = new HashMap<>()
+    // **Note** for systemProperties, settings, keystoreFiles etc:
+    // value could be a GString that is evaluated to just a String
+    // there are cases when value depends on task that is not executed yet on configuration stage
+    Map<String, Object> systemProperties = new HashMap<>()
 
     Map<String, Object> settings = new HashMap<>()
 
@@ -157,7 +160,7 @@ class ClusterConfiguration {
     List<Object> dependencies = new ArrayList<>()
 
     @Input
-    void systemProperty(String property, String value) {
+    void systemProperty(String property, Object value) {
         systemProperties.put(property, value)
     }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 0349130076c..4ede349b206 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -609,7 +609,6 @@ class ClusterFormationTasks {
 
     /** Adds a task to start an elasticsearch node with the given configuration */
     static Task configureStartTask(String name, Project project, Task setup, NodeInfo node) {
-
         // this closure is converted into ant nodes by groovy's AntBuilder
         Closure antRunner = { AntBuilder ant ->
             ant.exec(executable: node.executable, spawn: node.config.daemonize, dir: node.cwd, taskname: 'elasticsearch') {
@@ -630,13 +629,6 @@ class ClusterFormationTasks {
                 node.writeWrapperScript()
             }
 
-            // we must add debug options inside the closure so the config is read at execution time, as
-            // gradle task options are not processed until the end of the configuration phase
-            if (node.config.debug) {
-                println 'Running elasticsearch in debug mode, suspending until connected on port 8000'
-                node.env['ES_JAVA_OPTS'] = '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000'
-            }
-
             node.getCommandString().eachLine { line -> logger.info(line) }
 
             if (logger.isInfoEnabled() || node.config.daemonize == false) {
@@ -654,6 +646,27 @@ class ClusterFormationTasks {
         }
         start.doLast(elasticsearchRunner)
         start.doFirst {
+            // Configure ES JAVA OPTS - adds system properties, assertion flags, remote debug etc
+            List<String> esJavaOpts = [node.env.get('ES_JAVA_OPTS', '')]
+            String collectedSystemProperties = node.config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
+            esJavaOpts.add(collectedSystemProperties)
+            esJavaOpts.add(node.config.jvmArgs)
+            if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) {
+                // put the enable assertions options before other options to allow
+                // flexibility to disable assertions for specific packages or classes
+                // in the cluster-specific options
+                esJavaOpts.add("-ea")
+                esJavaOpts.add("-esa")
+            }
+            // we must add debug options inside the closure so the config is read at execution time, as
+            // gradle task options are not processed until the end of the configuration phase
+            if (node.config.debug) {
+                println 'Running elasticsearch in debug mode, suspending until connected on port 8000'
+                esJavaOpts.add('-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000')
+            }
+            node.env['ES_JAVA_OPTS'] = esJavaOpts.join(" ")
+
+            //
             project.logger.info("Starting node in ${node.clusterName} distribution: ${node.config.distribution}")
         }
         return start
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index 5e67dfa55cf..7844ea77fc1 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -180,15 +180,7 @@ class NodeInfo {
         }
 
         args.addAll("-E", "node.portsfile=true")
-        String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
-        String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
-        if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) {
-            // put the enable assertions options before other options to allow
-            // flexibility to disable assertions for specific packages or classes
-            // in the cluster-specific options
-            esJavaOpts = String.join(" ", "-ea", "-esa", esJavaOpts)
-        }
-        env = ['ES_JAVA_OPTS': esJavaOpts]
+        env = [:]
         for (Map.Entry<String, String> property : System.properties.entrySet()) {
             if (property.key.startsWith('tests.es.')) {
                 args.add("-E")
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 5af0a412b4c..225d523817e 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -89,18 +89,26 @@ String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporar
 String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
 String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
 
+String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
+String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
+
 // If all these variables are missing then we are testing against the internal fixture instead, which has the following
 // credentials hard-coded in.
 
-if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
+if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath
+        && !s3EC2Bucket && !s3EC2BasePath) {
   s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
   s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
   s3PermanentBucket = 'permanent-bucket-test'
   s3PermanentBasePath = 'integration_test'
 
+  s3EC2Bucket = 'ec2-bucket-test'
+  s3EC2BasePath = 'integration_test'
+
   useFixture = true
 
-} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
+} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath
+        || !s3EC2Bucket || !s3EC2BasePath) {
   throw new IllegalArgumentException("not all options specified to run against external S3 service")
 }
 
@@ -274,24 +282,52 @@ if (useFixture && minioDistribution) {
   integTestMinioRunner.dependsOn(startMinio)
   integTestMinioRunner.finalizedBy(stopMinio)
   // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
-  integTestMinioRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/30_repository_temporary_credentials/*'
+  integTestMinioRunner.systemProperty 'tests.rest.blacklist', [
+          'repository_s3/30_repository_temporary_credentials/*',
+          'repository_s3/40_repository_ec2_credentials/*'
+        ].join(",")
 
   project.check.dependsOn(integTestMinio)
 }
 
+File parentFixtures = new File(project.buildDir, "fixtures")
+File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
+
+task s3FixtureProperties {
+  outputs.file(s3FixtureFile)
+  def s3FixtureOptions = [
+      "tests.seed"                       : project.testSeed,
+      "s3Fixture.permanent_bucket_name"  : s3PermanentBucket,
+      "s3Fixture.permanent_key"          : s3PermanentAccessKey,
+      "s3Fixture.temporary_bucket_name"  : s3TemporaryBucket,
+      "s3Fixture.temporary_key"          : s3TemporaryAccessKey,
+      "s3Fixture.temporary_session_token": s3TemporarySessionToken,
+      "s3Fixture.ec2_bucket_name"        : s3EC2Bucket
+  ]
+
+  doLast {
+    file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n")
+  }
+}
+
 /** A task to start the AmazonS3Fixture which emulates an S3 service **/
 task s3Fixture(type: AntFixture) {
   dependsOn testClasses
+  dependsOn s3FixtureProperties
+  inputs.file(s3FixtureFile)
+
   env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
   executable = new File(project.runtimeJavaHome, 'bin/java')
-  args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket
+  args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3FixtureFile.getAbsolutePath()
 }
 
 Map<String, Object> expansions = [
   'permanent_bucket': s3PermanentBucket,
   'permanent_base_path': s3PermanentBasePath,
   'temporary_bucket': s3TemporaryBucket,
-  'temporary_base_path': s3TemporaryBasePath
+  'temporary_base_path': s3TemporaryBasePath,
+  'ec2_bucket': s3EC2Bucket,
+  'ec2_base_path': s3EC2BasePath
 ]
 
 processTestResources {
@@ -319,6 +355,10 @@ integTestCluster {
     /* Use a closure on the string to delay evaluation until tests are executed */
     setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}"
     setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}"
+    setting 's3.client.integration_test_ec2.endpoint', "http://${-> s3Fixture.addressAndPort}"
+
+    // to redirect InstanceProfileCredentialsProvider to custom auth point
+    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> s3Fixture.addressAndPort}"
   } else {
     println "Using an external service to test the repository-s3 plugin"
   }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java
index 9b38669da25..ce6c4723149 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java
@@ -18,6 +18,14 @@
  */
 package org.elasticsearch.repositories.s3;
 
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.common.TriFunction;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.test.fixture.AbstractHttpFixture;
 import com.amazonaws.util.DateUtils;
 
 import org.elasticsearch.common.Strings;
@@ -26,20 +34,26 @@ import org.elasticsearch.common.path.PathTrie;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.RestUtils;
-import org.elasticsearch.test.fixture.AbstractHttpFixture;
 
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
 
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLength;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Objects.requireNonNull;
 
 /**
  * {@link AmazonS3Fixture} emulates an AWS S3 service
@@ -47,63 +61,76 @@ import static java.nio.charset.StandardCharsets.UTF_8;
  * he implementation is based on official documentation available at https://docs.aws.amazon.com/AmazonS3/latest/API/.
  */
 public class AmazonS3Fixture extends AbstractHttpFixture {
+    private static final String AUTH = "AUTH";
+    private static final String NON_AUTH = "NON_AUTH";
+
+    private static final String EC2_PROFILE = "ec2Profile";
+
+    private final Properties properties;
+    private final Random random;
 
     /** List of the buckets stored on this test server **/
     private final Map<String, Bucket> buckets = ConcurrentCollections.newConcurrentMap();
 
     /** Request handlers for the requests made by the S3 client **/
     private final PathTrie<RequestHandler> handlers;
-    private final String permanentBucketName;
-    private final String temporaryBucketName;
 
     /**
      * Creates a {@link AmazonS3Fixture}
      */
-    private AmazonS3Fixture(final String workingDir, final String permanentBucketName, final String temporaryBucketName) {
+    private AmazonS3Fixture(final String workingDir, Properties properties) {
         super(workingDir);
-        this.permanentBucketName = permanentBucketName;
-        this.temporaryBucketName = temporaryBucketName;
+        this.properties = properties;
+        this.random = new Random(Long.parseUnsignedLong(requireNonNull(properties.getProperty("tests.seed")), 16));
 
-        this.buckets.put(permanentBucketName, new Bucket(permanentBucketName));
-        this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName));
-        this.handlers = defaultHandlers(buckets);
+        new Bucket("s3Fixture.permanent", false);
+        new Bucket("s3Fixture.temporary", true);
+        final Bucket ec2Bucket = new Bucket("s3Fixture.ec2",
+            randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10));
+
+        this.handlers = defaultHandlers(buckets, ec2Bucket);
+    }
+
+    private static String nonAuthPath(Request request) {
+        return nonAuthPath(request.getMethod(), request.getPath());
+    }
+
+    private static String nonAuthPath(String method, String path) {
+        return NON_AUTH + " " + method + " " + path;
+    }
+
+    private static String authPath(Request request) {
+        return authPath(request.getMethod(), request.getPath());
+    }
+
+    private static String authPath(String method, String path) {
+        return AUTH + " " + method + " " + path;
     }
 
     @Override
     protected Response handle(final Request request) throws IOException {
-        final RequestHandler handler = handlers.retrieve(request.getMethod() + " " + request.getPath(), request.getParameters());
+        final String nonAuthorizedPath = nonAuthPath(request);
+        final RequestHandler nonAuthorizedHandler = handlers.retrieve(nonAuthorizedPath, request.getParameters());
+        if (nonAuthorizedHandler != null) {
+            return nonAuthorizedHandler.handle(request);
+        }
+
+        final String authorizedPath = authPath(request);
+        final RequestHandler handler = handlers.retrieve(authorizedPath, request.getParameters());
         if (handler != null) {
-            final String authorization = request.getHeader("Authorization");
-            final String permittedBucket;
-            if (authorization.contains("s3_integration_test_permanent_access_key")) {
-                final String sessionToken = request.getHeader("x-amz-security-token");
-                if (sessionToken != null) {
-                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", "");
-                }
-                permittedBucket = permanentBucketName;
-            } else if (authorization.contains("s3_integration_test_temporary_access_key")) {
-                final String sessionToken = request.getHeader("x-amz-security-token");
-                if (sessionToken == null) {
-                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", "");
-                }
-                if (sessionToken.equals("s3_integration_test_temporary_session_token") == false) {
-                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", "");
-                }
-                permittedBucket = temporaryBucketName;
-            } else {
+            final String bucketName = request.getParam("bucket");
+            if (bucketName == null) {
                 return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", "");
             }
-
-            final String bucket = request.getParam("bucket");
-            if (bucket != null && permittedBucket.equals(bucket) == false) {
-                // allow a null bucket to support the multi-object-delete API which
-                // passes the bucket name in the host header instead of the URL.
-                if (buckets.containsKey(bucket)) {
-                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", "");
-                } else {
-                    return newBucketNotFoundError(request.getId(), bucket);
-                }
+            final Bucket bucket = buckets.get(bucketName);
+            if (bucket == null) {
+                return newBucketNotFoundError(request.getId(), bucketName);
             }
+            final Response authResponse = authenticateBucket(request, bucket);
+            if (authResponse != null) {
+                return authResponse;
+            }
+
             return handler.handle(request);
 
         } else {
@@ -111,24 +138,49 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         }
     }
 
-    public static void main(final String[] args) throws Exception {
-        if (args == null || args.length != 3) {
-            throw new IllegalArgumentException(
-                "AmazonS3Fixture <working directory> <bucket for permanent creds> <bucket for temporary creds>");
+    private Response authenticateBucket(Request request, Bucket bucket) {
+        final String authorization = request.getHeader("Authorization");
+        if (authorization == null) {
+            return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", "");
         }
+        if (authorization.contains(bucket.key)) {
+            final String sessionToken = request.getHeader("x-amz-security-token");
+            if (bucket.token == null) {
+                if (sessionToken != null) {
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", "");
+                }
+            } else {
+                if (sessionToken == null) {
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", "");
+                }
+                if (sessionToken.equals(bucket.token) == false) {
+                    return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", "");
+                }
+            }
+        }
+        return null;
+    }
 
-        final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1], args[2]);
+    public static void main(final String[] args) throws Exception {
+        if (args == null || args.length != 2) {
+            throw new IllegalArgumentException("AmazonS3Fixture <working directory> <property file>");
+        }
+        final Properties properties = new Properties();
+        try (InputStream is = Files.newInputStream(PathUtils.get(args[1]))) {
+            properties.load(is);
+        }
+        final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], properties);
         fixture.listen();
     }
 
     /** Builds the default request handlers **/
-    private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> buckets) {
+    private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> buckets, final Bucket ec2Bucket) {
         final PathTrie<RequestHandler> handlers = new PathTrie<>(RestUtils.REST_DECODER);
 
         // HEAD Object
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
-        objectsPaths("HEAD /{bucket}").forEach(path ->
+        objectsPaths(authPath(HttpHead.METHOD_NAME, "/{bucket}")).forEach(path ->
             handlers.insert(path, (request) -> {
                 final String bucketName = request.getParam("bucket");
 
@@ -150,7 +202,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         // PUT Object
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
-        objectsPaths("PUT /{bucket}").forEach(path ->
+        objectsPaths(authPath(HttpPut.METHOD_NAME, "/{bucket}")).forEach(path ->
             handlers.insert(path, (request) -> {
                 final String destBucketName = request.getParam("bucket");
 
@@ -200,7 +252,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         // DELETE Object
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
-        objectsPaths("DELETE /{bucket}").forEach(path ->
+        objectsPaths(authPath(HttpDelete.METHOD_NAME, "/{bucket}")).forEach(path ->
             handlers.insert(path, (request) -> {
                 final String bucketName = request.getParam("bucket");
 
@@ -218,7 +270,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         // GET Object
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
-        objectsPaths("GET /{bucket}").forEach(path ->
+        objectsPaths(authPath(HttpGet.METHOD_NAME, "/{bucket}")).forEach(path ->
             handlers.insert(path, (request) -> {
                 final String bucketName = request.getParam("bucket");
 
@@ -239,7 +291,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         // HEAD Bucket
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketHEAD.html
-        handlers.insert("HEAD /{bucket}", (request) -> {
+        handlers.insert(authPath(HttpHead.METHOD_NAME, "/{bucket}"), (request) -> {
             String bucket = request.getParam("bucket");
             if (Strings.hasText(bucket) && buckets.containsKey(bucket)) {
                 return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
@@ -251,7 +303,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         // GET Bucket (List Objects) Version 1
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
-        handlers.insert("GET /{bucket}/", (request) -> {
+        handlers.insert(authPath(HttpGet.METHOD_NAME, "/{bucket}/"), (request) -> {
             final String bucketName = request.getParam("bucket");
 
             final Bucket bucket = buckets.get(bucketName);
@@ -269,7 +321,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         // Delete Multiple Objects
         //
         // https://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-        handlers.insert("POST /", (request) -> {
+        handlers.insert(nonAuthPath(HttpPost.METHOD_NAME, "/"), (request) -> {
             final List<String> deletes = new ArrayList<>();
             final List<String> errors = new ArrayList<>();
 
@@ -292,7 +344,12 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
 
                                 boolean found = false;
                                 for (Bucket bucket : buckets.values()) {
-                                    if (bucket.objects.remove(objectName) != null) {
+                                    if (bucket.objects.containsKey(objectName)) {
+                                        final Response authResponse = authenticateBucket(request, bucket);
+                                        if (authResponse != null) {
+                                            return authResponse;
+                                        }
+                                        bucket.objects.remove(objectName);
                                         found = true;
                                     }
                                 }
@@ -311,23 +368,80 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
             return newInternalError(request.getId(), "Something is wrong with this POST multiple deletes request");
         });
 
+        // non-authorized requests
+
+        TriFunction<String, String, String, Response> credentialResponseFunction = (profileName, key, token) -> {
+            final Date expiration = new Date(new Date().getTime() + TimeUnit.DAYS.toMillis(1));
+            final String response = "{"
+                 + "\"AccessKeyId\": \"" + key + "\","
+                 + "\"Expiration\": \"" + DateUtils.formatISO8601Date(expiration) + "\","
+                 + "\"RoleArn\": \"" + randomAsciiAlphanumOfLengthBetween(random, 1, 20) + "\","
+                 + "\"SecretAccessKey\": \"" + randomAsciiAlphanumOfLengthBetween(random, 1, 20) + "\","
+                 + "\"Token\": \"" + token + "\""
+                 + "}";
+
+            final Map<String, String> headers = new HashMap<>(contentType("application/json"));
+            return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8));
+        };
+
+        // GET
+        //
+        // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
+        handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/"), (request) -> {
+            final String response = EC2_PROFILE;
+
+            final Map<String, String> headers = new HashMap<>(contentType("text/plain"));
+            return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8));
+        });
+
+        // GET
+        //
+        // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
+        handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/{profileName}"), (request) -> {
+            final String profileName = request.getParam("profileName");
+            if (EC2_PROFILE.equals(profileName) == false) {
+                return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown credentials".getBytes(UTF_8));
+            }
+            return credentialResponseFunction.apply(profileName, ec2Bucket.key, ec2Bucket.token);
+        });
+
         return handlers;
     }
 
+    private static String prop(Properties properties, String propertyName) {
+        return requireNonNull(properties.getProperty(propertyName),
+            "property '" + propertyName + "' is missing");
+    }
+
     /**
      * Represents a S3 bucket.
      */
-    static class Bucket {
+    class Bucket {
 
         /** Bucket name **/
         final String name;
 
+        final String key;
+
+        final String token;
+
         /** Blobs contained in the bucket **/
         final Map<String, byte[]> objects;
 
-        Bucket(final String name) {
-            this.name = Objects.requireNonNull(name);
+        private Bucket(final String prefix, final boolean tokenRequired) {
+            this(prefix, prop(properties, prefix + "_key"),
+                tokenRequired ? prop(properties, prefix + "_session_token") : null);
+        }
+
+        private Bucket(final String prefix, final String key, final String token) {
+            this.name = prop(properties, prefix + "_bucket_name");
+            this.key = key;
+            this.token = token;
+
             this.objects = ConcurrentCollections.newConcurrentMap();
+            if (buckets.put(name, this) != null) {
+                throw new IllegalArgumentException("bucket " + name + " is already registered");
+            }
         }
     }
 
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
new file mode 100644
index 00000000000..2df3b8290a1
--- /dev/null
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
@@ -0,0 +1,243 @@
+# Integration tests for repository-s3
+
+---
+setup:
+
+  # Register repository with ec2 credentials
+  - do:
+      snapshot.create_repository:
+        repository: repository_ec2
+        body:
+          type: s3
+          settings:
+            bucket: ${ec2_bucket}
+            client: integration_test_ec2
+            base_path: ${ec2_base_path}
+            canned_acl: private
+            storage_class: standard
+
+---
+"Snapshot and Restore with repository-s3 using ec2 credentials":
+
+  # Get repository
+  - do:
+      snapshot.get_repository:
+        repository: repository_ec2
+
+  - match: { repository_ec2.settings.bucket : ${ec2_bucket} }
+  - match: { repository_ec2.settings.client : "integration_test_ec2" }
+  - match: { repository_ec2.settings.base_path : ${ec2_base_path} }
+  - match: { repository_ec2.settings.canned_acl : "private" }
+  - match: { repository_ec2.settings.storage_class : "standard" }
+  - is_false: repository_ec2.settings.access_key
+  - is_false: repository_ec2.settings.secret_key
+  - is_false: repository_ec2.settings.session_token
+
+  # Index documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    1
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    2
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    3
+          - snapshot: one
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Create a first snapshot
+  - do:
+      snapshot.create:
+        repository: repository_ec2
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-one }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.include_global_state: true }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.status:
+        repository: repository_ec2
+        snapshot: snapshot-one
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: snapshot-one }
+  - match: { snapshots.0.state : SUCCESS }
+
+  # Index more documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    4
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    5
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    6
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    7
+          - snapshot: two
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Create a second snapshot
+  - do:
+      snapshot.create:
+        repository: repository_ec2
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-two }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.get:
+        repository: repository_ec2
+        snapshot: snapshot-one,snapshot-two
+
+  - is_true: snapshots
+  - match: { snapshots.0.state : SUCCESS }
+  - match: { snapshots.1.state : SUCCESS }
+
+  # Delete the index
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the second snapshot
+  - do:
+      snapshot.restore:
+        repository: repository_ec2
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Delete the index again
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the first snapshot
+  - do:
+      snapshot.restore:
+        repository: repository_ec2
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Remove the snapshots
+  - do:
+      snapshot.delete:
+        repository: repository_ec2
+        snapshot: snapshot-two
+
+  - do:
+      snapshot.delete:
+        repository: repository_ec2
+        snapshot: snapshot-one
+
+---
+"Register a repository with a non existing bucket":
+
+  - do:
+      catch: /repository_exception/
+      snapshot.create_repository:
+        repository: repository_ec2
+        body:
+          type: s3
+          settings:
+            bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+            client: integration_test_temporary
+
+---
+"Register a repository with a non existing client":
+
+  - do:
+      catch: /repository_exception/
+      snapshot.create_repository:
+        repository: repository_ec2
+        body:
+          type: s3
+          settings:
+            bucket: repository_ec2
+            client: unknown
+
+---
+"Get a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.get:
+        repository: repository_ec2
+        snapshot: missing
+
+---
+"Delete a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.delete:
+        repository: repository_ec2
+        snapshot: missing
+
+---
+"Restore a non existing snapshot":
+
+  - do:
+      catch: /snapshot_restore_exception/
+      snapshot.restore:
+        repository: repository_ec2
+        snapshot: missing
+        wait_for_completion: true
+
+---
+teardown:
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: repository_ec2

From 1924f5d07c9f739821bc8c749a5314a074dab36b Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Wed, 18 Jul 2018 12:42:07 +0200
Subject: [PATCH 082/260] Add more contexts to painless execute api (#30511)

This change adds two contexts the execute scripts against:

* SEARCH_SCRIPT: Allows to run scripts in a search script context.
This context is used in `function_score` query's script function,
script fields, script sorting and `terms_set` query.

* FILTER_SCRIPT: Allows to run scripts in a filter script context.
This context is used in the `script` query.

In both contexts a index name needs to be specified and a sample document.
The document is needed to create an in-memory index that the script can
access via the `doc[...]` and other notations. The index name is needed
because a mapping is needed to index the document.

Examples:

```
POST /_scripts/painless/_execute
{
  "script": {
    "source": "doc['field'].value.length()"
  },
  "context" : {
    "search_script": {
      "document": {
        "field": "four"
      },
      "index": "my-index"
    }
  }
}
```

Returns:

```
{
  "result": 4
}
```

POST /_scripts/painless/_execute
{
  "script": {
    "source": "doc['field'].value.length() <= params.max_length",
    "params": {
      "max_length": 4
    }
  },
  "context" : {
    "filter_script": {
      "document": {
        "field": "four"
      },
      "index": "my-index"
    }
  }
}

Returns:

```
{
  "result": true
}
```

Also changed PainlessExecuteAction.TransportAction to use TransportSingleShardAction
instead of HandledAction, because now in case score or filter contexts are used
the request needs to be redirected to a node that has an active IndexService
for the index being referenced (a node with a shard copy for that index).
---
 .../painless/painless-execute-script.asciidoc | 133 +++++-
 .../painless/PainlessExecuteAction.java       | 439 +++++++++++++++---
 .../painless/PainlessExecuteApiTests.java     | 113 +++++
 .../painless/PainlessExecuteRequestTests.java |  44 +-
 .../painless/70_execute_painless_scripts.yml  |  52 ++-
 5 files changed, 694 insertions(+), 87 deletions(-)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessExecuteApiTests.java

diff --git a/docs/painless/painless-execute-script.asciidoc b/docs/painless/painless-execute-script.asciidoc
index a3ac5b578d7..2aca9597786 100644
--- a/docs/painless/painless-execute-script.asciidoc
+++ b/docs/painless/painless-execute-script.asciidoc
@@ -9,23 +9,24 @@ The Painless execute API allows an arbitrary script to be executed and a result
 .Parameters
 [options="header"]
 |======
-| Name             | Required  | Default                | Description
-| `script`         | yes       | -                      | The script to execute
-| `context`        | no        | `painless_test`        | The context the script should be executed in.
+| Name              | Required  | Default                | Description
+| `script`          | yes       | -                      | The script to execute
+| `context`         | no        | `painless_test`        | The context the script should be executed in.
+| `context_setup`   | no        | -                      | Additional parameters to the context.
 |======
 
 ==== Contexts
 
 Contexts control how scripts are executed, what variables are available at runtime and what the return type is.
 
-===== Painless test script context
+===== Painless test context
 
 The `painless_test` context executes scripts as is and do not add any special parameters.
 The only variable that is available is `params`, which can be used to access user defined values.
 The result of the script is always converted to a string.
 If no context is specified then this context is used by default.
 
-==== Example
+====== Example
 
 Request:
 
@@ -52,4 +53,124 @@ Response:
   "result": "0.1"
 }
 --------------------------------------------------
-// TESTRESPONSE
\ No newline at end of file
+// TESTRESPONSE
+
+===== Filter context
+
+The `filter` context executes scripts as if they were executed inside a `script` query.
+For testing purposes a document must be provided that will be indexed temporarily in-memory and
+is accessible to the script being tested. Because of this the _source, stored fields and doc values
+are available in the script being tested.
+
+The following parameters may be specified in `context_setup` for a filter context:
+
+document:: Contains the document that will be temporarily indexed in-memory and is accessible from the script.
+index:: The name of an index containing a mapping that is compatable with the document being indexed.
+
+====== Example
+
+[source,js]
+----------------------------------------------------------------
+PUT /my-index
+{
+  "mappings": {
+    "_doc": {
+      "properties": {
+        "field": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+
+POST /_scripts/painless/_execute
+{
+  "script": {
+    "source": "doc['field'].value.length() <= params.max_length",
+    "params": {
+      "max_length": 4
+    }
+  },
+  "context": "filter",
+  "context_setup": {
+    "index": "my-index",
+    "document": {
+      "field": "four"
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+  "result": true
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+
+===== Score context
+
+The `score` context executes scripts as if they were executed inside a `script_score` function in
+`function_score` query.
+
+The following parameters may be specified in `context_setup` for a score context:
+
+document:: Contains the document that will be temporarily indexed in-memory and is accessible from the script.
+index:: The name of an index containing a mapping that is compatable with the document being indexed.
+query:: If `_score` is used in the script then a query can specified that will be used to compute a score.
+
+====== Example
+
+[source,js]
+----------------------------------------------------------------
+PUT /my-index
+{
+  "mappings": {
+    "_doc": {
+      "properties": {
+        "field": {
+          "type": "keyword"
+        },
+        "rank": {
+          "type": "long"
+        }
+      }
+    }
+  }
+}
+
+
+POST /_scripts/painless/_execute
+{
+  "script": {
+    "source": "doc['rank'].value / params.max_rank",
+    "params": {
+      "max_rank": 5.0
+    }
+  },
+  "context": "score",
+  "context_setup": {
+    "index": "my-index",
+    "document": {
+      "rank": 4
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+  "result": 0.8
+}
+--------------------------------------------------
+// TESTRESPONSE
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExecuteAction.java
index 01139f6cf2e..229c919a2e6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExecuteAction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExecuteAction.java
@@ -18,41 +18,75 @@
  */
 package org.elasticsearch.painless;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.RAMDirectory;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.single.shard.SingleShardRequest;
+import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedBiFunction;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.script.FilterScript;
+import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.Locale;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -75,40 +109,181 @@ public class PainlessExecuteAction extends Action<PainlessExecuteAction.Response
         return new Response();
     }
 
-    public static class Request extends ActionRequest implements ToXContent {
+    public static class Request extends SingleShardRequest<Request> implements ToXContent {
 
         private static final ParseField SCRIPT_FIELD = new ParseField("script");
         private static final ParseField CONTEXT_FIELD = new ParseField("context");
+        private static final ParseField CONTEXT_SETUP_FIELD = new ParseField("context_setup");
         private static final ConstructingObjectParser<Request, Void> PARSER = new ConstructingObjectParser<>(
-            "painless_execute_request", args -> new Request((Script) args[0], (SupportedContext) args[1]));
+            "painless_execute_request", args -> new Request((Script) args[0], (String) args[1], (ContextSetup) args[2]));
 
         static {
             PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> Script.parse(p), SCRIPT_FIELD);
-            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
-                // For now only accept an empty json object:
-                XContentParser.Token token = p.nextToken();
-                assert token == XContentParser.Token.FIELD_NAME;
-                String contextType = p.currentName();
-                token = p.nextToken();
-                assert token == XContentParser.Token.START_OBJECT;
-                token = p.nextToken();
-                assert token == XContentParser.Token.END_OBJECT;
-                token = p.nextToken();
-                assert token == XContentParser.Token.END_OBJECT;
-                return SupportedContext.valueOf(contextType.toUpperCase(Locale.ROOT));
-            }, CONTEXT_FIELD);
+            PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), CONTEXT_FIELD);
+            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ContextSetup::parse, CONTEXT_SETUP_FIELD);
+        }
+
+        static final Map<String, ScriptContext<?>> SUPPORTED_CONTEXTS;
+
+        static {
+            Map<String, ScriptContext<?>> supportedContexts = new HashMap<>();
+            supportedContexts.put("painless_test", PainlessTestScript.CONTEXT);
+            supportedContexts.put("filter", FilterScript.CONTEXT);
+            supportedContexts.put("score", ScoreScript.CONTEXT);
+            SUPPORTED_CONTEXTS = Collections.unmodifiableMap(supportedContexts);
+        }
+
+        static ScriptContext<?> fromScriptContextName(String name) {
+            ScriptContext<?> scriptContext = SUPPORTED_CONTEXTS.get(name);
+            if (scriptContext == null) {
+                throw new UnsupportedOperationException("unsupported script context name [" + name + "]");
+            }
+            return scriptContext;
+        }
+
+        static class ContextSetup implements Writeable, ToXContentObject {
+
+            private static final ParseField INDEX_FIELD = new ParseField("index");
+            private static final ParseField DOCUMENT_FIELD = new ParseField("document");
+            private static final ParseField QUERY_FIELD = new ParseField("query");
+            private static final ConstructingObjectParser<ContextSetup, Void> PARSER =
+                new ConstructingObjectParser<>("execute_script_context",
+                    args -> new ContextSetup((String) args[0], (BytesReference) args[1], (QueryBuilder) args[2]));
+
+            static {
+                PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), INDEX_FIELD);
+                PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+                    try (XContentBuilder b = XContentBuilder.builder(p.contentType().xContent())) {
+                        b.copyCurrentStructure(p);
+                        return BytesReference.bytes(b);
+                    }
+                }, DOCUMENT_FIELD);
+                PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
+                    AbstractQueryBuilder.parseInnerQueryBuilder(p), QUERY_FIELD);
+            }
+
+            private final String index;
+            private final BytesReference document;
+            private final QueryBuilder query;
+
+            private XContentType xContentType;
+
+            static ContextSetup parse(XContentParser parser, Void context) throws IOException {
+                ContextSetup contextSetup = PARSER.parse(parser, null);
+                contextSetup.setXContentType(parser.contentType());
+                return contextSetup;
+            }
+
+            ContextSetup(String index, BytesReference document, QueryBuilder query) {
+                this.index = index;
+                this.document = document;
+                this.query = query;
+            }
+
+            ContextSetup(StreamInput in) throws IOException {
+                index = in.readOptionalString();
+                document = in.readOptionalBytesReference();
+                String xContentType = in.readOptionalString();
+                if (xContentType  != null) {
+                    this.xContentType = XContentType.fromMediaType(xContentType);
+                }
+                query = in.readOptionalNamedWriteable(QueryBuilder.class);
+            }
+
+            public String getIndex() {
+                return index;
+            }
+
+            public BytesReference getDocument() {
+                return document;
+            }
+
+            public QueryBuilder getQuery() {
+                return query;
+            }
+
+            public XContentType getXContentType() {
+                return xContentType;
+            }
+
+            public void setXContentType(XContentType xContentType) {
+                this.xContentType = xContentType;
+            }
+
+            @Override
+            public boolean equals(Object o) {
+                if (this == o) return true;
+                if (o == null || getClass() != o.getClass()) return false;
+                ContextSetup that = (ContextSetup) o;
+                return Objects.equals(index, that.index) &&
+                    Objects.equals(document, that.document) &&
+                    Objects.equals(query, that.query);
+            }
+
+            @Override
+            public int hashCode() {
+                return Objects.hash(index, document, query);
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                out.writeOptionalString(index);
+                out.writeOptionalBytesReference(document);
+                out.writeOptionalString(xContentType != null ? xContentType.mediaType(): null);
+                out.writeOptionalNamedWriteable(query);
+            }
+
+            @Override
+            public String toString() {
+                return "ContextSetup{" +
+                    ", index='" + index + '\'' +
+                    ", document=" + document +
+                    ", query=" + query +
+                    ", xContentType=" + xContentType +
+                    '}';
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder.startObject();
+                {
+                    if (index != null) {
+                        builder.field(INDEX_FIELD.getPreferredName(), index);
+                    }
+                    if (document != null) {
+                        builder.field(DOCUMENT_FIELD.getPreferredName());
+                        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
+                            LoggingDeprecationHandler.INSTANCE, document, xContentType)) {
+                            builder.generator().copyCurrentStructure(parser);
+                        }
+                    }
+                    if (query != null) {
+                        builder.field(QUERY_FIELD.getPreferredName(), query);
+                    }
+                }
+                builder.endObject();
+                return builder;
+            }
+
         }
 
         private Script script;
-        private SupportedContext context;
+        private ScriptContext<?> context = PainlessTestScript.CONTEXT;
+        private ContextSetup contextSetup;
 
         static Request parse(XContentParser parser) throws IOException {
             return PARSER.parse(parser, null);
         }
 
-        Request(Script script, SupportedContext context) {
+        Request(Script script, String scriptContextName, ContextSetup setup) {
             this.script = Objects.requireNonNull(script);
-            this.context = context != null ? context : SupportedContext.PAINLESS_TEST;
+            if (scriptContextName != null) {
+                this.context = fromScriptContextName(scriptContextName);
+            }
+            if (setup != null) {
+                this.contextSetup = setup;
+                index(contextSetup.index);
+            }
         }
 
         Request() {
@@ -118,16 +293,28 @@ public class PainlessExecuteAction extends Action<PainlessExecuteAction.Response
             return script;
         }
 
-        public SupportedContext getContext() {
+        public ScriptContext<?> getContext() {
             return context;
         }
 
+        public ContextSetup getContextSetup() {
+            return contextSetup;
+        }
+
         @Override
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
             if (script.getType() != ScriptType.INLINE) {
                 validationException = addValidationError("only inline scripts are supported", validationException);
             }
+            if (needDocumentAndIndex(context)) {
+                if (contextSetup.index == null) {
+                    validationException = addValidationError("index is a required parameter for current context", validationException);
+                }
+                if (contextSetup.document == null) {
+                    validationException = addValidationError("document is a required parameter for current context", validationException);
+                }
+            }
             return validationException;
         }
 
@@ -135,26 +322,35 @@ public class PainlessExecuteAction extends Action<PainlessExecuteAction.Response
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             script = new Script(in);
-            context = SupportedContext.fromId(in.readByte());
+            if (in.getVersion().onOrBefore(Version.V_6_4_0)) {
+                byte scriptContextId = in.readByte();
+                assert scriptContextId == 0;
+            } else {
+                context = fromScriptContextName(in.readString());
+                contextSetup = in.readOptionalWriteable(ContextSetup::new);
+            }
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             script.writeTo(out);
-            out.writeByte(context.id);
+            if (out.getVersion().onOrBefore(Version.V_6_4_0)) {
+                out.writeByte((byte) 0);
+            } else {
+                out.writeString(context.name);
+                out.writeOptionalWriteable(contextSetup);
+            }
         }
 
         // For testing only:
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field(SCRIPT_FIELD.getPreferredName(), script);
-            builder.startObject(CONTEXT_FIELD.getPreferredName());
-            {
-                builder.startObject(context.name());
-                builder.endObject();
+            builder.field(CONTEXT_FIELD.getPreferredName(), context.name);
+            if (contextSetup != null) {
+                builder.field(CONTEXT_SETUP_FIELD.getPreferredName(), contextSetup);
             }
-            builder.endObject();
             return builder;
         }
 
@@ -164,41 +360,28 @@ public class PainlessExecuteAction extends Action<PainlessExecuteAction.Response
             if (o == null || getClass() != o.getClass()) return false;
             Request request = (Request) o;
             return Objects.equals(script, request.script) &&
-                context == request.context;
+                Objects.equals(context, request.context) &&
+                Objects.equals(contextSetup, request.contextSetup);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(script, context);
+            return Objects.hash(script, context, contextSetup);
         }
 
-        public enum SupportedContext {
-
-            PAINLESS_TEST((byte) 0);
-
-            private final byte id;
-
-            SupportedContext(byte id) {
-                this.id = id;
-            }
-
-            public static SupportedContext fromId(byte id) {
-                switch (id) {
-                    case 0:
-                        return PAINLESS_TEST;
-                    default:
-                        throw new IllegalArgumentException("unknown context [" + id + "]");
-                }
-            }
+        @Override
+        public String toString() {
+            return "Request{" +
+                "script=" + script +
+                "context=" + context +
+                ", contextSetup=" + contextSetup +
+                '}';
         }
 
-    }
-
-    public static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
-
-        RequestBuilder(ElasticsearchClient client) {
-            super(client, INSTANCE, new Request());
+        static boolean needDocumentAndIndex(ScriptContext<?> scriptContext) {
+            return scriptContext == FilterScript.CONTEXT || scriptContext == ScoreScript.CONTEXT;
         }
+
     }
 
     public static class Response extends ActionResponse implements ToXContentObject {
@@ -274,31 +457,139 @@ public class PainlessExecuteAction extends Action<PainlessExecuteAction.Response
 
     }
 
-    public static class TransportAction extends HandledTransportAction<Request, Response> {
-
+    public static class TransportAction extends TransportSingleShardAction<Request, Response> {
 
         private final ScriptService scriptService;
+        private final IndicesService indicesServices;
 
         @Inject
-        public TransportAction(Settings settings, TransportService transportService,
-                               ActionFilters actionFilters, ScriptService scriptService) {
-            super(settings, NAME, transportService, actionFilters, Request::new);
+        public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                               ScriptService scriptService, ClusterService clusterService, IndicesService indicesServices) {
+            super(settings, NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
+                // Forking a thread here, because only light weight operations should happen on network thread and
+                // Creating a in-memory index is not light weight
+                // TODO: is MANAGEMENT TP the right TP? Right now this is an admin api (see action name).
+                Request::new, ThreadPool.Names.MANAGEMENT);
             this.scriptService = scriptService;
+            this.indicesServices = indicesServices;
         }
+
         @Override
-        protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-            switch (request.context) {
-                case PAINLESS_TEST:
-                    PainlessTestScript.Factory factory = scriptService.compile(request.script, PainlessTestScript.CONTEXT);
-                    PainlessTestScript painlessTestScript = factory.newInstance(request.script.getParams());
-                    String result = Objects.toString(painlessTestScript.execute());
-                    listener.onResponse(new Response(result));
-                    break;
-                default:
-                    throw new UnsupportedOperationException("unsupported context [" + request.context + "]");
+        protected Response newResponse() {
+            return new Response();
+        }
+
+        @Override
+        protected ClusterBlockException checkRequestBlock(ClusterState state, InternalRequest request) {
+            if (request.concreteIndex() != null) {
+                return super.checkRequestBlock(state, request);
+            }
+            return null;
+        }
+
+        @Override
+        protected boolean resolveIndex(Request request) {
+            return request.contextSetup != null && request.contextSetup.getIndex() != null;
+        }
+
+        @Override
+        protected ShardsIterator shards(ClusterState state, InternalRequest request) {
+            if (request.concreteIndex() == null) {
+                return null;
+            }
+            return state.routingTable().index(request.concreteIndex()).randomAllActiveShardsIt();
+        }
+
+        @Override
+        protected Response shardOperation(Request request, ShardId shardId) throws IOException {
+            IndexService indexService;
+            if (request.contextSetup != null && request.contextSetup.getIndex() != null) {
+                ClusterState clusterState = clusterService.state();
+                IndicesOptions indicesOptions = IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+                String indexExpression = request.contextSetup.index;
+                Index[] concreteIndices =
+                    indexNameExpressionResolver.concreteIndices(clusterState, indicesOptions, indexExpression);
+                if (concreteIndices.length != 1) {
+                    throw new IllegalArgumentException("[" + indexExpression + "] does not resolve to a single index");
+                }
+                Index concreteIndex = concreteIndices[0];
+                indexService = indicesServices.indexServiceSafe(concreteIndex);
+            } else {
+                indexService = null;
+            }
+            return innerShardOperation(request, scriptService, indexService);
+        }
+
+        static Response innerShardOperation(Request request, ScriptService scriptService, IndexService indexService) throws IOException {
+            final ScriptContext<?> scriptContext = request.context;
+            if (scriptContext == PainlessTestScript.CONTEXT) {
+                PainlessTestScript.Factory factory = scriptService.compile(request.script, PainlessTestScript.CONTEXT);
+                PainlessTestScript painlessTestScript = factory.newInstance(request.script.getParams());
+                String result = Objects.toString(painlessTestScript.execute());
+                return new Response(result);
+            } else if (scriptContext == FilterScript.CONTEXT) {
+                return prepareRamIndex(request, (context, leafReaderContext) -> {
+                    FilterScript.Factory factory = scriptService.compile(request.script, FilterScript.CONTEXT);
+                    FilterScript.LeafFactory leafFactory =
+                        factory.newFactory(request.getScript().getParams(), context.lookup());
+                    FilterScript filterScript = leafFactory.newInstance(leafReaderContext);
+                    filterScript.setDocument(0);
+                    boolean result = filterScript.execute();
+                    return new Response(result);
+                }, indexService);
+            } else if (scriptContext == ScoreScript.CONTEXT) {
+                return prepareRamIndex(request, (context, leafReaderContext) -> {
+                    ScoreScript.Factory factory = scriptService.compile(request.script, ScoreScript.CONTEXT);
+                    ScoreScript.LeafFactory leafFactory =
+                        factory.newFactory(request.getScript().getParams(), context.lookup());
+                    ScoreScript scoreScript = leafFactory.newInstance(leafReaderContext);
+                    scoreScript.setDocument(0);
+
+                    if (request.contextSetup.query != null) {
+                        Query luceneQuery = request.contextSetup.query.rewrite(context).toQuery(context);
+                        IndexSearcher indexSearcher = new IndexSearcher(leafReaderContext.reader());
+                        luceneQuery = indexSearcher.rewrite(luceneQuery);
+                        Weight weight = indexSearcher.createWeight(luceneQuery, true, 1f);
+                        Scorer scorer = weight.scorer(indexSearcher.getIndexReader().leaves().get(0));
+                        // Consume the first (and only) match.
+                        int docID = scorer.iterator().nextDoc();
+                        assert docID == scorer.docID();
+                        scoreScript.setScorer(scorer);
+                    }
+
+                    double result = scoreScript.execute();
+                    return new Response(result);
+                }, indexService);
+            } else {
+                throw new UnsupportedOperationException("unsupported context [" + scriptContext.name + "]");
             }
         }
 
+        private static Response prepareRamIndex(Request request,
+                                                CheckedBiFunction<QueryShardContext, LeafReaderContext, Response, IOException> handler,
+                                                IndexService indexService) throws IOException {
+
+            Analyzer defaultAnalyzer = indexService.getIndexAnalyzers().getDefaultIndexAnalyzer();
+
+            try (RAMDirectory ramDirectory = new RAMDirectory()) {
+                try (IndexWriter indexWriter = new IndexWriter(ramDirectory, new IndexWriterConfig(defaultAnalyzer))) {
+                    String index = indexService.index().getName();
+                    String type = indexService.mapperService().documentMapper().type();
+                    BytesReference document = request.contextSetup.document;
+                    XContentType xContentType = request.contextSetup.xContentType;
+                    SourceToParse sourceToParse = SourceToParse.source(index, type, "_id", document, xContentType);
+                    ParsedDocument parsedDocument = indexService.mapperService().documentMapper().parse(sourceToParse);
+                    indexWriter.addDocuments(parsedDocument.docs());
+                    try (IndexReader indexReader = DirectoryReader.open(indexWriter)) {
+                        final long absoluteStartMillis = System.currentTimeMillis();
+                        QueryShardContext context =
+                            indexService.newQueryShardContext(0, indexReader, () -> absoluteStartMillis, null);
+                        return handler.apply(context, indexReader.leaves().get(0));
+                    }
+                }
+            }
+        }
     }
 
     static class RestAction extends BaseRestHandler {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessExecuteApiTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessExecuteApiTests.java
new file mode 100644
index 00000000000..ce92a224f4e
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessExecuteApiTests.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.painless;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.painless.PainlessExecuteAction.Request;
+import org.elasticsearch.painless.PainlessExecuteAction.Response;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.painless.PainlessExecuteAction.TransportAction.innerShardOperation;
+import static org.hamcrest.Matchers.equalTo;
+
+public class PainlessExecuteApiTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singleton(PainlessPlugin.class);
+    }
+
+    public void testDefaults() throws IOException {
+        ScriptService scriptService = getInstanceFromNode(ScriptService.class);
+        Request request = new Request(new Script("100.0 / 1000.0"), null, null);
+        Response response = innerShardOperation(request, scriptService, null);
+        assertThat(response.getResult(), equalTo("0.1"));
+
+        Map<String, Object> params = new HashMap<>();
+        params.put("count", 100.0D);
+        params.put("total", 1000.0D);
+        request = new Request(new Script(ScriptType.INLINE, "painless", "params.count / params.total", params), null, null);
+        response = innerShardOperation(request, scriptService, null);
+        assertThat(response.getResult(), equalTo("0.1"));
+
+        Exception e = expectThrows(ScriptException.class,
+            () -> {
+            Request r = new Request(new Script(ScriptType.INLINE,
+                "painless", "params.count / params.total + doc['constant']", params), null, null);
+            innerShardOperation(r, scriptService, null);
+        });
+        assertThat(e.getCause().getMessage(), equalTo("Variable [doc] is not defined."));
+    }
+
+    public void testFilterExecutionContext() throws IOException {
+        ScriptService scriptService = getInstanceFromNode(ScriptService.class);
+        IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "field", "type=long");
+
+        Request.ContextSetup contextSetup = new Request.ContextSetup("index", new BytesArray("{\"field\": 3}"), null);
+        contextSetup.setXContentType(XContentType.JSON);
+        Request request = new Request(new Script("doc['field'].value >= 3"), "filter", contextSetup);
+        Response response = innerShardOperation(request, scriptService, indexService);
+        assertThat(response.getResult(), equalTo(true));
+
+        contextSetup = new Request.ContextSetup("index", new BytesArray("{\"field\": 3}"), null);
+        contextSetup.setXContentType(XContentType.JSON);
+        request = new Request(new Script(ScriptType.INLINE, "painless", "doc['field'].value >= params.max",
+            singletonMap("max", 3)), "filter", contextSetup);
+        response = innerShardOperation(request, scriptService, indexService);
+        assertThat(response.getResult(), equalTo(true));
+
+        contextSetup = new Request.ContextSetup("index", new BytesArray("{\"field\": 2}"), null);
+        contextSetup.setXContentType(XContentType.JSON);
+        request = new Request(new Script(ScriptType.INLINE, "painless", "doc['field'].value >= params.max",
+            singletonMap("max", 3)), "filter", contextSetup);
+        response = innerShardOperation(request, scriptService, indexService);
+        assertThat(response.getResult(), equalTo(false));
+    }
+
+    public void testScoreExecutionContext() throws IOException {
+        ScriptService scriptService = getInstanceFromNode(ScriptService.class);
+        IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "rank", "type=long", "text", "type=text");
+
+        Request.ContextSetup contextSetup = new Request.ContextSetup("index",
+            new BytesArray("{\"rank\": 4.0, \"text\": \"quick brown fox\"}"), new MatchQueryBuilder("text", "fox"));
+        contextSetup.setXContentType(XContentType.JSON);
+        Request request = new Request(new Script(ScriptType.INLINE, "painless",
+            "Math.round((_score + (doc['rank'].value / params.max_rank)) * 100.0) / 100.0", singletonMap("max_rank", 5.0)), "score",
+            contextSetup);
+        Response response = innerShardOperation(request, scriptService, indexService);
+        assertThat(response.getResult(), equalTo(1.09D));
+    }
+
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessExecuteRequestTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessExecuteRequestTests.java
index 488ae0e1643..44cd6b5304d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessExecuteRequestTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessExecuteRequestTests.java
@@ -18,9 +18,18 @@
  */
 package org.elasticsearch.painless;
 
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.painless.PainlessExecuteAction.Request.ContextSetup;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractStreamableXContentTestCase;
 
 import java.io.IOException;
@@ -28,12 +37,22 @@ import java.util.Collections;
 
 public class PainlessExecuteRequestTests extends AbstractStreamableXContentTestCase<PainlessExecuteAction.Request> {
 
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, false, Collections.emptyList()).getNamedWriteables());
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, Collections.emptyList()).getNamedXContents());
+    }
+
     @Override
     protected PainlessExecuteAction.Request createTestInstance() {
         Script script = new Script(randomAlphaOfLength(10));
-        PainlessExecuteAction.Request.SupportedContext context = randomBoolean() ?
-            PainlessExecuteAction.Request.SupportedContext.PAINLESS_TEST : null;
-        return new PainlessExecuteAction.Request(script, context);
+        ScriptContext<?> context = randomBoolean() ? randomFrom(PainlessExecuteAction.Request.SUPPORTED_CONTEXTS.values()) : null;
+        ContextSetup contextSetup = randomBoolean() ? randomContextSetup() : null;
+        return new PainlessExecuteAction.Request(script, context != null ? context.name : null, contextSetup);
     }
 
     @Override
@@ -53,9 +72,26 @@ public class PainlessExecuteRequestTests extends AbstractStreamableXContentTestC
 
     public void testValidate() {
         Script script = new Script(ScriptType.STORED, null, randomAlphaOfLength(10), Collections.emptyMap());
-        PainlessExecuteAction.Request request = new PainlessExecuteAction.Request(script, null);
+        PainlessExecuteAction.Request request = new PainlessExecuteAction.Request(script, null, null);
         Exception e = request.validate();
         assertNotNull(e);
         assertEquals("Validation Failed: 1: only inline scripts are supported;", e.getMessage());
     }
+
+    private static ContextSetup randomContextSetup() {
+        String index = randomBoolean() ? randomAlphaOfLength(4) : null;
+        QueryBuilder query = randomBoolean() ? new MatchAllQueryBuilder() : null;
+        // TODO: pass down XContextType to createTestInstance() method.
+        // otherwise the document itself is different causing test failures.
+        // This should be done in a seperate change as the test instance is created before xcontent type is randomly picked and
+        // all the createTestInstance() methods need to be changed, which will make this a big chnage
+//        BytesReference doc = randomBoolean() ? new BytesArray("{}") : null;
+        BytesReference doc = null;
+
+        ContextSetup contextSetup = new ContextSetup(index, doc, query);
+//        if (doc != null) {
+//            contextSetup.setXContentType(XContentType.JSON);
+//        }
+        return contextSetup;
+    }
 }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml
index 7b915cc38db..1e34a776189 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml
@@ -1,3 +1,18 @@
+setup:
+  - do:
+      indices.create:
+        index: my-index
+        body:
+          mappings:
+            doc:
+              properties:
+                rank:
+                  type: long
+                field:
+                  type: keyword
+                text:
+                  type: text
+
 ---
 "Execute with defaults":
   - do:
@@ -11,7 +26,7 @@
   - match: { result: "0.1" }
 
 ---
-"Execute with execute_api_script context":
+"Execute with painless_test context":
   - do:
       scripts_painless_execute:
         body:
@@ -20,6 +35,37 @@
             params:
               var1: 10
               var2: 100
-          context:
-            painless_test: {}
+          context: "painless_test"
   - match: { result: "-90" }
+
+---
+"Execute with filter context":
+  - do:
+      scripts_painless_execute:
+        body:
+          script:
+            source: "doc['field'].value.length() <= params.max_length"
+            params:
+              max_length: 4
+          context: "filter"
+          context_setup:
+            document:
+              field: "four"
+            index: "my-index"
+  - match: { result: true }
+
+---
+"Execute with score context":
+  - do:
+      scripts_painless_execute:
+        body:
+          script:
+            source: "doc['rank'].value / params.max_rank"
+            params:
+              max_rank: 5.0
+          context: "score"
+          context_setup:
+            document:
+              rank: 4
+            index: "my-index"
+  - match: { result: 0.8 }

From 53ab470264261b25e799696085c134df09ddaf77 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Wed, 18 Jul 2018 13:33:57 +0200
Subject: [PATCH 083/260] use before instead of onOrBefore

---
 .../org/elasticsearch/painless/PainlessExecuteAction.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExecuteAction.java
index 229c919a2e6..094a62d188b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExecuteAction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExecuteAction.java
@@ -322,7 +322,7 @@ public class PainlessExecuteAction extends Action<PainlessExecuteAction.Response
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             script = new Script(in);
-            if (in.getVersion().onOrBefore(Version.V_6_4_0)) {
+            if (in.getVersion().before(Version.V_6_4_0)) {
                 byte scriptContextId = in.readByte();
                 assert scriptContextId == 0;
             } else {
@@ -335,7 +335,7 @@ public class PainlessExecuteAction extends Action<PainlessExecuteAction.Response
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             script.writeTo(out);
-            if (out.getVersion().onOrBefore(Version.V_6_4_0)) {
+            if (out.getVersion().before(Version.V_6_4_0)) {
                 out.writeByte((byte) 0);
             } else {
                 out.writeString(context.name);

From 380b45b96590a313507f26ff2b0e3a2af54f4706 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Wed, 18 Jul 2018 12:58:17 +0100
Subject: [PATCH 084/260] Improve docs for search preferences (#32159)

Today it is unclear what guarantees are offered by the search preference
feature, and we claim a guarantee that is stronger than what we really offer:

> A custom value will be used to guarantee that the same shards will be used
> for the same custom value.

This commit clarifies this documentation.

Forward-port of #32098 to `master`.
---
 .../search/request/preference.asciidoc        | 67 +++++++++++++------
 1 file changed, 45 insertions(+), 22 deletions(-)

diff --git a/docs/reference/search/request/preference.asciidoc b/docs/reference/search/request/preference.asciidoc
index 4fd801c5f76..5f3fcb2efa6 100644
--- a/docs/reference/search/request/preference.asciidoc
+++ b/docs/reference/search/request/preference.asciidoc
@@ -1,38 +1,55 @@
 [[search-request-preference]]
 === Preference
 
-Controls a `preference` of which shard copies on which to execute the
-search. By default, the operation is randomized among the available shard 
-copies, unless allocation awareness is used.
+Controls a `preference` of the shard copies on which to execute the search.  By
+default, Elasticsearch selects from the available shard copies in an
+unspecified order, taking the <<allocation-awareness,allocation awareness>> and
+<<search-adaptive-replica,adaptive replica selection>> configuration into
+account. However, it may sometimes be desirable to try and route certain
+searches to certain sets of shard copies, for instance to make better use of
+per-copy caches.
 
 The `preference` is a query string parameter which can be set to:
 
 [horizontal]
-`_local`:: 
-	The operation will prefer to be executed on a local
-	allocated shard if possible.
+`_only_local`::
+	The operation will be executed only on shards allocated to the local
+	node.
+
+`_local`::
+	The operation will be executed on shards allocated to the local node if
+	possible, and will fall back to other shards if not.
 
 `_prefer_nodes:abc,xyz`::
-	Prefers execution on the nodes with the provided
-	node ids (`abc` or `xyz` in this case) if applicable.
+	The operation will be executed on nodes with one of the provided node
+	ids (`abc` or `xyz` in this case) if possible. If suitable shard copies
+	exist on more than one of the selected nodes then the order of
+	preference between these copies is unspecified.
 
-`_shards:2,3`:: 
-	Restricts the operation to the specified shards. (`2`
-	and `3` in this case). This preference can be combined with other
-	preferences but it has to appear first: `_shards:2,3|_local`
+`_shards:2,3`::
+	Restricts the operation to the specified shards. (`2` and `3` in this
+	case).  This preference can be combined with other preferences but it
+	has to appear first: `_shards:2,3|_local`
 
-`_only_nodes`::
-    Restricts the operation to nodes specified in <<cluster,node specification>>
+`_only_nodes:abc*,x*yz,...`::
+	Restricts the operation to nodes specified according to the
+	<<cluster,node specification>>. If suitable shard copies exist on more
+	than one of the selected nodes then the order of preference between
+	these copies is unspecified.
 
-Custom (string) value:: 
-	A custom value will be used to guarantee that
-	the same shards will be used for the same custom value. This can help
-	with "jumping values" when hitting different shards in different refresh
-	states. A sample value can be something like the web session id, or the
-	user name.
+Custom (string) value::
+	Any value that does not start with `_`. If two searches both give the same
+	custom string value for their preference and the underlying cluster state
+	does not change then the same ordering of shards will be used for the
+	searches. This does not guarantee that the exact same shards will be used
+	each time: the cluster state, and therefore the selected shards, may change
+	for a number of reasons including shard relocations and shard failures, and
+	nodes may sometimes reject searches causing fallbacks to alternative nodes.
+	However, in practice the ordering of shards tends to remain stable for long
+	periods of time. A good candidate for a custom preference value is something
+	like the web session id or the user name.
 
-For instance, use the user's session ID to ensure consistent ordering of results
-for the user:
+For instance, use the user's session ID `xyzabc123` as follows:
 
 [source,js]
 ------------------------------------------------
@@ -47,3 +64,9 @@ GET /_search?preference=xyzabc123
 ------------------------------------------------
 // CONSOLE
 
+NOTE: The `_only_local` preference guarantees only to use shard copies on the
+local node, which is sometimes useful for troubleshooting. All other options do
+not _fully_ guarantee that any particular shard copies are used in a search,
+and on a changing index this may mean that repeated searches may yield
+different results if they are executed on different shard copies which are in
+different refresh states.

From 6de1f96cadd8d187e3732b710eb10264f733d422 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Wed, 18 Jul 2018 15:32:36 +0200
Subject: [PATCH 085/260] Fix BwC Tests looking for UUID Pre 6.4 (#32158)

* UUID field was added for #31791 and only went into 6.4 and 7.0
* Fixes #32119
---
 qa/mixed-cluster/build.gradle                                 | 1 -
 .../resources/rest-api-spec/test/indices.stats/10_index.yml   | 4 ++++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
index db743cb12b1..ac57d51def7 100644
--- a/qa/mixed-cluster/build.gradle
+++ b/qa/mixed-cluster/build.gradle
@@ -57,7 +57,6 @@ for (Version version : bwcVersions.wireCompatible) {
   tasks.getByName("${baseName}#mixedClusterTestRunner").configure {
     /* To support taking index snapshots, we have to set path.repo setting */
     systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
-    systemProperty 'tests.rest.blacklist', ['indices.stats/10_index/Index - all'].join(',')
   }
 }
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml
index 42847b05cd1..564a482727f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/10_index.yml
@@ -41,6 +41,10 @@ setup:
 
 ---
 "Index - all":
+  - skip:
+      version: " - 6.3.99"
+      reason: "uuid is only available from 6.4.0 on"
+
   - do:
       indices.stats: { index: _all }
 

From cfb30144c9175d1f1538af0aa5c1841f72159128 Mon Sep 17 00:00:00 2001
From: Alan Woodward <romseygeek@gmail.com>
Date: Wed, 18 Jul 2018 14:43:20 +0100
Subject: [PATCH 086/260] Call setReferences() on custom referring tokenfilters
 in _analyze (#32157)

When building custom tokenfilters without an index in the _analyze endpoint,
we need to ensure that referring filters are correctly built by calling
their #setReferences() method

Fixes #32154
---
 .../test/analysis-common/40_token_filters.yml | 15 +++++++++++
 .../analyze/TransportAnalyzeAction.java       | 26 ++++++++++++++++++-
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
index 3dca3bfd777..150fa39dcb9 100644
--- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
+++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
@@ -1557,3 +1557,18 @@
             filter:    [my_bengali_stem]
     - length: { tokens: 1 }
     - match:  { tokens.0.token: কর }
+
+---
+"multiplexer":
+    - do:
+        indices.analyze:
+          body:
+            text: "The quick fox"
+            tokenizer: "standard"
+            filter:
+              - type: multiplexer
+                filters: [ lowercase, uppercase ]
+                preserve_original: false
+    - length: { tokens: 6 }
+    - match: { tokens.0.token: the }
+    - match: { tokens.1.token: THE }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
index 35f1f725b65..5c5da62571f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
@@ -52,6 +52,7 @@ import org.elasticsearch.index.analysis.CustomAnalyzerProvider;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.analysis.ReferringFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -574,6 +575,7 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                                                                 Environment environment, Tuple<String, TokenizerFactory> tokenizerFactory,
                                                                 List<CharFilterFactory> charFilterFactoryList, boolean normalizer) throws IOException {
         List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
+        List<ReferringFilterFactory> referringFilters = new ArrayList<>();
         if (request.tokenFilters() != null && request.tokenFilters().size() > 0) {
             List<AnalyzeRequest.NameOrDefinition> tokenFilters = request.tokenFilters();
             for (AnalyzeRequest.NameOrDefinition tokenFilter : tokenFilters) {
@@ -594,7 +596,9 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                     tokenFilterFactory = tokenFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenfilter", settings);
                     tokenFilterFactory = CustomAnalyzerProvider.checkAndApplySynonymFilter(tokenFilterFactory, tokenizerFactory.v1(), tokenizerFactory.v2(), tokenFilterFactoryList,
                         charFilterFactoryList, environment);
-
+                    if (tokenFilterFactory instanceof ReferringFilterFactory) {
+                        referringFilters.add((ReferringFilterFactory)tokenFilterFactory);
+                    }
 
                 } else {
                     AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory;
@@ -629,6 +633,26 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                 tokenFilterFactoryList.add(tokenFilterFactory);
             }
         }
+        if (referringFilters.isEmpty() == false) {
+            // The request included at least one custom referring tokenfilter that has not already been built by the
+            // analysis registry, so we need to set its references.  Note that this will only apply pre-built
+            // tokenfilters
+            if (indexSettings == null) {
+                Settings settings = Settings.builder()
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                    .build();
+                IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
+                indexSettings = new IndexSettings(metaData, Settings.EMPTY);
+            }
+            Map<String, TokenFilterFactory> prebuiltFilters = analysisRegistry.buildTokenFilterFactories(indexSettings);
+            for (ReferringFilterFactory rff : referringFilters) {
+                rff.setReferences(prebuiltFilters);
+            }
+
+        }
         return tokenFilterFactoryList;
     }
 

From 605dc49c4828fe74d039e361cbe283bbf19a9a49 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Wed, 18 Jul 2018 09:29:52 -0700
Subject: [PATCH 087/260] Painless: Fix caching bug and clean up
 addPainlessClass. (#32142)

This change cleans up the addPainlessClass methods by doing the following things:

* Rename many variable names to match the new conventions described in the JavaDocs
for PainlessLookup
* Decouples Whitelist.Class from adding a PainlessClass directly
* Adds a second version of addPainlessClass that is intended for use to add future
defaults in a follow PR

This change also fixes the method and field caches by storing Classes instead of Strings
since it would technically be possible now that the whitelists are extendable to have
different Classes with the same name. It was convenient to add this change together
since some of the new constants are shared.

Note the changes are largely mechanical again where all the code behavior should
remain the same.
---
 .../lookup/PainlessLookupBuilder.java         | 527 +++++++++++-------
 1 file changed, 327 insertions(+), 200 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 2150c0b210a..ecf15c7ad2c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -30,236 +30,249 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Stack;
 import java.util.regex.Pattern;
 
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_PAINLESS_CLASS_NAME;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.anyTypeNameToPainlessTypeName;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
 
 public class PainlessLookupBuilder {
-    private static final Pattern TYPE_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$");
 
-    private static final Map<String, PainlessMethod> methodCache = new HashMap<>();
-    private static final Map<String, PainlessField> fieldCache = new HashMap<>();
+    private static class PainlessMethodCacheKey {
 
-    private static String buildMethodCacheKey(String structName, String methodName, List<Class<?>> arguments) {
-        StringBuilder key = new StringBuilder();
-        key.append(structName);
-        key.append(methodName);
+        private final Class<?> javaClass;
+        private final String methodName;
+        private final List<Class<?>> painlessTypeParameters;
 
-        for (Class<?> argument : arguments) {
-            key.append(argument.getName());
+        private PainlessMethodCacheKey(Class<?> javaClass, String methodName, List<Class<?>> painlessTypeParameters) {
+            this.javaClass = javaClass;
+            this.methodName = methodName;
+            this.painlessTypeParameters = Collections.unmodifiableList(painlessTypeParameters);
         }
 
-        return key.toString();
+        @Override
+        public boolean equals(Object object) {
+            if (this == object) {
+                return true;
+            }
+
+            if (object == null || getClass() != object.getClass()) {
+                return false;
+            }
+
+            PainlessMethodCacheKey that = (PainlessMethodCacheKey)object;
+
+            return Objects.equals(javaClass, that.javaClass)   &&
+                   Objects.equals(methodName, that.methodName) &&
+                   Objects.equals(painlessTypeParameters, that.painlessTypeParameters);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(javaClass, methodName, painlessTypeParameters);
+        }
     }
 
-    private static String buildFieldCacheKey(String structName, String fieldName, String typeName) {
-        return structName + fieldName + typeName;
+    private static class PainlessFieldCacheKey {
+
+        private final Class<?> javaClass;
+        private final String fieldName;
+        private final Class<?> painlessType;
+
+        private PainlessFieldCacheKey(Class<?> javaClass, String fieldName, Class<?> painlessType) {
+            this.javaClass = javaClass;
+            this.fieldName = fieldName;
+            this.painlessType = painlessType;
+        }
+
+        @Override
+        public boolean equals(Object object) {
+            if (this == object) {
+                return true;
+            }
+
+            if (object == null || getClass() != object.getClass()) {
+                return false;
+            }
+
+            PainlessFieldCacheKey that = (PainlessFieldCacheKey) object;
+
+            return Objects.equals(javaClass, that.javaClass) &&
+                   Objects.equals(fieldName, that.fieldName) &&
+                   Objects.equals(painlessType, that.painlessType);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(javaClass, fieldName, painlessType);
+        }
     }
 
-    private final Map<String, Class<?>> painlessTypesToJavaClasses;
+    private static final Map<PainlessMethodCacheKey, PainlessMethod> painlessMethodCache = new HashMap<>();
+    private static final Map<PainlessFieldCacheKey,  PainlessField>  painlessFieldCache  = new HashMap<>();
+
+    private static final Pattern CLASS_NAME_PATTERN  = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$");
+    private static final Pattern METHOD_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][_a-zA-Z0-9]*$");
+    private static final Pattern FIELD_NAME_PATTERN  = Pattern.compile("^[_a-zA-Z][_a-zA-Z0-9]*$");
+
+    private static String anyTypesArrayToCanonicalString(Class<?>[] anyTypesArray, boolean toPainlessTypes) {
+        return anyTypesListToCanonicalString(Arrays.asList(anyTypesArray), toPainlessTypes);
+    }
+
+    private static String anyTypesListToCanonicalString(List<Class<?>> anyTypesList, boolean toPainlessTypes) {
+        StringBuilder anyTypesCanonicalStringBuilder = new StringBuilder("[");
+
+        int anyTypesSize = anyTypesList.size();
+        int anyTypesIndex = 0;
+
+        for (Class<?> anyType : anyTypesList) {
+            String anyTypeCanonicalName = anyType.getCanonicalName();
+
+            if (toPainlessTypes) {
+                anyTypeCanonicalName = anyTypeNameToPainlessTypeName(anyTypeCanonicalName);
+            }
+
+            anyTypesCanonicalStringBuilder.append(anyTypeCanonicalName);
+
+            if (++anyTypesIndex < anyTypesSize) {
+                anyTypesCanonicalStringBuilder.append(",");
+            }
+        }
+
+        anyTypesCanonicalStringBuilder.append("]");
+
+        return anyTypesCanonicalStringBuilder.toString();
+    }
+
+    private final List<Whitelist> whitelists;
+
+    private final Map<String, Class<?>> painlessClassNamesToJavaClasses;
     private final Map<Class<?>, PainlessClassBuilder> javaClassesToPainlessClassBuilders;
 
     public PainlessLookupBuilder(List<Whitelist> whitelists) {
-        painlessTypesToJavaClasses = new HashMap<>();
+        this.whitelists = whitelists;
+
+        painlessClassNamesToJavaClasses = new HashMap<>();
         javaClassesToPainlessClassBuilders = new HashMap<>();
 
-        String origin = null;
-
-        painlessTypesToJavaClasses.put("def", def.class);
-        javaClassesToPainlessClassBuilders.put(def.class, new PainlessClassBuilder("def", Object.class, Type.getType(Object.class)));
-
-        try {
-            // first iteration collects all the Painless type names that
-            // are used for validation during the second iteration
-            for (Whitelist whitelist : whitelists) {
-                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
-                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-                    PainlessClassBuilder painlessStruct =
-                            javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(painlessTypeName));
-
-                    if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
-                        throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
-                            "[" + painlessStruct.clazz.getName() + "] and [" + whitelistStruct.javaClassName + "]");
-                    }
-
-                    origin = whitelistStruct.origin;
-                    addStruct(whitelist.javaClassLoader, whitelistStruct);
-
-                    painlessStruct = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(painlessTypeName));
-                    javaClassesToPainlessClassBuilders.put(painlessStruct.clazz, painlessStruct);
-                }
-            }
-
-            // second iteration adds all the constructors, methods, and fields that will
-            // be available in Painless along with validating they exist and all their types have
-            // been white-listed during the first iteration
-            for (Whitelist whitelist : whitelists) {
-                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
-                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-
-                    for (WhitelistConstructor whitelistConstructor : whitelistStruct.whitelistConstructors) {
-                        origin = whitelistConstructor.origin;
-                        addConstructor(painlessTypeName, whitelistConstructor);
-                    }
-
-                    for (WhitelistMethod whitelistMethod : whitelistStruct.whitelistMethods) {
-                        origin = whitelistMethod.origin;
-                        addMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod);
-                    }
-
-                    for (WhitelistField whitelistField : whitelistStruct.whitelistFields) {
-                        origin = whitelistField.origin;
-                        addField(painlessTypeName, whitelistField);
-                    }
-                }
-            }
-        } catch (Exception exception) {
-            throw new IllegalArgumentException("error loading whitelist(s) " + origin, exception);
-        }
-
-        // goes through each Painless struct and determines the inheritance list,
-        // and then adds all inherited types to the Painless struct's whitelist
-        for (Class<?> javaClass : javaClassesToPainlessClassBuilders.keySet()) {
-            PainlessClassBuilder painlessStruct = javaClassesToPainlessClassBuilders.get(javaClass);
-
-            List<String> painlessSuperStructs = new ArrayList<>();
-            Class<?> javaSuperClass = painlessStruct.clazz.getSuperclass();
-
-            Stack<Class<?>> javaInteraceLookups = new Stack<>();
-            javaInteraceLookups.push(painlessStruct.clazz);
-
-            // adds super classes to the inheritance list
-            if (javaSuperClass != null && javaSuperClass.isInterface() == false) {
-                while (javaSuperClass != null) {
-                    PainlessClassBuilder painlessSuperStruct = javaClassesToPainlessClassBuilders.get(javaSuperClass);
-
-                    if (painlessSuperStruct != null) {
-                        painlessSuperStructs.add(painlessSuperStruct.name);
-                    }
-
-                    javaInteraceLookups.push(javaSuperClass);
-                    javaSuperClass = javaSuperClass.getSuperclass();
-                }
-            }
-
-            // adds all super interfaces to the inheritance list
-            while (javaInteraceLookups.isEmpty() == false) {
-                Class<?> javaInterfaceLookup = javaInteraceLookups.pop();
-
-                for (Class<?> javaSuperInterface : javaInterfaceLookup.getInterfaces()) {
-                    PainlessClassBuilder painlessInterfaceStruct = javaClassesToPainlessClassBuilders.get(javaSuperInterface);
-
-                    if (painlessInterfaceStruct != null) {
-                        String painlessInterfaceStructName = painlessInterfaceStruct.name;
-
-                        if (painlessSuperStructs.contains(painlessInterfaceStructName) == false) {
-                            painlessSuperStructs.add(painlessInterfaceStructName);
-                        }
-
-                        for (Class<?> javaPushInterface : javaInterfaceLookup.getInterfaces()) {
-                            javaInteraceLookups.push(javaPushInterface);
-                        }
-                    }
-                }
-            }
-
-            // copies methods and fields from super structs to the parent struct
-            copyStruct(painlessStruct.name, painlessSuperStructs);
-
-            // copies methods and fields from Object into interface types
-            if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) {
-                PainlessClassBuilder painlessObjectStruct = javaClassesToPainlessClassBuilders.get(Object.class);
-
-                if (painlessObjectStruct != null) {
-                    copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name));
-                }
-            }
-        }
-
-        // precompute runtime classes
-        for (PainlessClassBuilder painlessStruct : javaClassesToPainlessClassBuilders.values()) {
-            addRuntimeClass(painlessStruct);
-        }
+        painlessClassNamesToJavaClasses.put(DEF_PAINLESS_CLASS_NAME, def.class);
+        javaClassesToPainlessClassBuilders.put(def.class,
+                new PainlessClassBuilder(DEF_PAINLESS_CLASS_NAME, Object.class, Type.getType(Object.class)));
     }
 
-    private void addStruct(ClassLoader whitelistClassLoader, WhitelistClass whitelistStruct) {
-        String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-        String importedPainlessTypeName = painlessTypeName;
+    private Class<?> painlessTypeNameToPainlessType(String painlessTypeName) {
+        return PainlessLookupUtility.painlessTypeNameToPainlessType(painlessTypeName, painlessClassNamesToJavaClasses);
+    }
 
-        if (TYPE_NAME_PATTERN.matcher(painlessTypeName).matches() == false) {
-            throw new IllegalArgumentException("invalid struct type name [" + painlessTypeName + "]");
+    private void validatePainlessType(Class<?> painlessType) {
+        PainlessLookupUtility.validatePainlessType(painlessType, javaClassesToPainlessClassBuilders.keySet());
+    }
+
+    public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importPainlessClassName) {
+        Objects.requireNonNull(classLoader);
+        Objects.requireNonNull(javaClassName);
+
+        String painlessClassName = anyTypeNameToPainlessTypeName(javaClassName);
+
+        if (CLASS_NAME_PATTERN.matcher(painlessClassName).matches() == false) {
+            throw new IllegalArgumentException("invalid painless class name [" + painlessClassName + "]");
         }
 
-        int index = whitelistStruct.javaClassName.lastIndexOf('.');
-
-        if (index != -1) {
-            importedPainlessTypeName = whitelistStruct.javaClassName.substring(index + 1).replace('$', '.');
-        }
+        String importedPainlessClassName = anyTypeNameToPainlessTypeName(javaClassName.substring(javaClassName.lastIndexOf('.') + 1));
 
         Class<?> javaClass;
 
-        if      ("void".equals(whitelistStruct.javaClassName))    javaClass = void.class;
-        else if ("boolean".equals(whitelistStruct.javaClassName)) javaClass = boolean.class;
-        else if ("byte".equals(whitelistStruct.javaClassName))    javaClass = byte.class;
-        else if ("short".equals(whitelistStruct.javaClassName))   javaClass = short.class;
-        else if ("char".equals(whitelistStruct.javaClassName))    javaClass = char.class;
-        else if ("int".equals(whitelistStruct.javaClassName))     javaClass = int.class;
-        else if ("long".equals(whitelistStruct.javaClassName))    javaClass = long.class;
-        else if ("float".equals(whitelistStruct.javaClassName))   javaClass = float.class;
-        else if ("double".equals(whitelistStruct.javaClassName))  javaClass = double.class;
+        if ("void".equals(javaClassName)) javaClass = void.class;
+        else if ("boolean".equals(javaClassName)) javaClass = boolean.class;
+        else if ("byte".equals(javaClassName)) javaClass = byte.class;
+        else if ("short".equals(javaClassName)) javaClass = short.class;
+        else if ("char".equals(javaClassName)) javaClass = char.class;
+        else if ("int".equals(javaClassName)) javaClass = int.class;
+        else if ("long".equals(javaClassName)) javaClass = long.class;
+        else if ("float".equals(javaClassName)) javaClass = float.class;
+        else if ("double".equals(javaClassName)) javaClass = double.class;
         else {
             try {
-                javaClass = Class.forName(whitelistStruct.javaClassName, true, whitelistClassLoader);
+                javaClass = Class.forName(javaClassName, true, classLoader);
+
+                if (javaClass == def.class) {
+                    throw new IllegalArgumentException("cannot add reserved painless class [" + DEF_PAINLESS_CLASS_NAME + "]");
+                }
+
+                if (javaClass.isArray()) {
+                    throw new IllegalArgumentException("cannot add an array type java class [" + javaClassName + "] as a painless class");
+                }
             } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("invalid java class name [" + whitelistStruct.javaClassName + "]" +
-                    " for struct [" + painlessTypeName + "]");
+                throw new IllegalArgumentException("java class [" + javaClassName + "] not found", cnfe);
             }
         }
 
-        PainlessClassBuilder existingStruct = javaClassesToPainlessClassBuilders.get(javaClass);
+        addPainlessClass(painlessClassName, importedPainlessClassName, javaClass, importPainlessClassName);
+    }
 
-        if (existingStruct == null) {
-            PainlessClassBuilder struct = new PainlessClassBuilder(painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass));
-            painlessTypesToJavaClasses.put(painlessTypeName, javaClass);
-            javaClassesToPainlessClassBuilders.put(javaClass, struct);
-        } else if (existingStruct.clazz.equals(javaClass) == false) {
-            throw new IllegalArgumentException("struct [" + painlessTypeName + "] is used to " +
-                "illegally represent multiple java classes [" + whitelistStruct.javaClassName + "] and " +
-                "[" + existingStruct.clazz.getName() + "]");
+    public void addPainlessClass(Class<?> javaClass, boolean importPainlessClassName) {
+        Objects.requireNonNull(javaClass);
+
+        if (javaClass == def.class) {
+            throw new IllegalArgumentException("cannot specify reserved painless class [" + DEF_PAINLESS_CLASS_NAME + "]");
         }
 
-        if (painlessTypeName.equals(importedPainlessTypeName)) {
-            if (whitelistStruct.onlyFQNJavaClassName == false) {
-                throw new IllegalArgumentException("must use only_fqn parameter on type [" + painlessTypeName + "] with no package");
+        String javaClassName = javaClass.getCanonicalName();
+        String painlessClassName = anyTypeNameToPainlessTypeName(javaClassName);
+        String importedPainlessClassName = anyTypeNameToPainlessTypeName(javaClassName.substring(javaClassName.lastIndexOf('.') + 1));
+
+        addPainlessClass(painlessClassName, importedPainlessClassName, javaClass, importPainlessClassName);
+    }
+
+    private void addPainlessClass(
+        String painlessClassName, String importedPainlessClassName, Class<?> javaClass, boolean importPainlessClassName) {
+        PainlessClassBuilder existingPainlessClassBuilder = javaClassesToPainlessClassBuilders.get(javaClass);
+
+        if (existingPainlessClassBuilder == null) {
+            PainlessClassBuilder painlessClassBuilder = new PainlessClassBuilder(painlessClassName, javaClass, Type.getType(javaClass));
+            painlessClassNamesToJavaClasses.put(painlessClassName, javaClass);
+            javaClassesToPainlessClassBuilders.put(javaClass, painlessClassBuilder);
+        } else if (existingPainlessClassBuilder.clazz.equals(javaClass) == false) {
+            throw new IllegalArgumentException("painless class [" + painlessClassName + "] illegally represents multiple java classes " +
+                    "[" + javaClass.getCanonicalName() + "] and [" + existingPainlessClassBuilder.clazz.getCanonicalName() + "]");
+        }
+
+        if (painlessClassName.equals(importedPainlessClassName)) {
+            if (importPainlessClassName == true) {
+                throw new IllegalArgumentException(
+                        "must use only_fqn parameter on painless class [" + painlessClassName + "] with no package");
             }
         } else {
-            Class<?> importedJavaClass = painlessTypesToJavaClasses.get(importedPainlessTypeName);
+            Class<?> importedJavaClass = painlessClassNamesToJavaClasses.get(importedPainlessClassName);
 
             if (importedJavaClass == null) {
-                if (whitelistStruct.onlyFQNJavaClassName == false) {
-                    if (existingStruct != null) {
-                        throw new IllegalArgumentException("inconsistent only_fqn parameters found for type [" + painlessTypeName + "]");
+                if (importPainlessClassName) {
+                    if (existingPainlessClassBuilder != null) {
+                        throw new IllegalArgumentException(
+                                "inconsistent only_fqn parameters found for painless class [" + painlessClassName + "]");
                     }
 
-                    painlessTypesToJavaClasses.put(importedPainlessTypeName, javaClass);
+                    painlessClassNamesToJavaClasses.put(importedPainlessClassName, javaClass);
                 }
             } else if (importedJavaClass.equals(javaClass) == false) {
-                throw new IllegalArgumentException("imported name [" + painlessTypeName + "] is used to " +
-                    "illegally represent multiple java classes [" + whitelistStruct.javaClassName + "] " +
-                    "and [" + importedJavaClass.getName() + "]");
-            } else if (whitelistStruct.onlyFQNJavaClassName) {
-                throw new IllegalArgumentException("inconsistent only_fqn parameters found for type [" + painlessTypeName + "]");
+                throw new IllegalArgumentException("painless class [" + importedPainlessClassName + "] illegally represents multiple " +
+                        "java classes [" + javaClass.getCanonicalName() + "] and [" + importedJavaClass.getCanonicalName() + "]");
+            } else if (importPainlessClassName == false) {
+                throw new IllegalArgumentException(
+                        "inconsistent only_fqn parameters found for painless class [" + painlessClassName + "]");
             }
         }
     }
 
     private void addConstructor(String ownerStructName, WhitelistConstructor whitelistConstructor) {
-        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for constructor with " +
@@ -273,7 +286,7 @@ public class PainlessLookupBuilder {
             String painlessParameterTypeName = whitelistConstructor.painlessParameterTypeNames.get(parameterCount);
 
             try {
-                Class<?> painlessParameterClass = getJavaClassFromPainlessType(painlessParameterTypeName);
+                Class<?> painlessParameterClass = painlessTypeNameToPainlessType(painlessParameterTypeName);
 
                 painlessParametersTypes.add(painlessParameterClass);
                 javaClassParameters[parameterCount] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(painlessParameterClass);
@@ -307,7 +320,8 @@ public class PainlessLookupBuilder {
                     " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames);
             }
 
-            painlessConstructor = methodCache.computeIfAbsent(buildMethodCacheKey(ownerStruct.name, "<init>", painlessParametersTypes),
+            painlessConstructor = painlessMethodCache.computeIfAbsent(
+                new PainlessMethodCacheKey(ownerStruct.clazz, "<init>", painlessParametersTypes),
                 key -> new PainlessMethod("<init>", ownerStruct.clazz, null, void.class, painlessParametersTypes,
                     asmConstructor, javaConstructor.getModifiers(), javaHandle));
             ownerStruct.constructors.put(painlessMethodKey, painlessConstructor);
@@ -319,14 +333,14 @@ public class PainlessLookupBuilder {
     }
 
     private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, WhitelistMethod whitelistMethod) {
-        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
                 "name [" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
         }
 
-        if (TYPE_NAME_PATTERN.matcher(whitelistMethod.javaMethodName).matches() == false) {
+        if (METHOD_NAME_PATTERN.matcher(whitelistMethod.javaMethodName).matches() == false) {
             throw new IllegalArgumentException("invalid method name" +
                 " [" + whitelistMethod.javaMethodName + "] for owner struct [" + ownerStructName + "].");
         }
@@ -358,7 +372,7 @@ public class PainlessLookupBuilder {
             String painlessParameterTypeName = whitelistMethod.painlessParameterTypeNames.get(parameterCount);
 
             try {
-                Class<?> painlessParameterClass = getJavaClassFromPainlessType(painlessParameterTypeName);
+                Class<?> painlessParameterClass = painlessTypeNameToPainlessType(painlessParameterTypeName);
 
                 painlessParametersTypes.add(painlessParameterClass);
                 javaClassParameters[parameterCount + augmentedOffset] =
@@ -384,7 +398,7 @@ public class PainlessLookupBuilder {
         Class<?> painlessReturnClass;
 
         try {
-            painlessReturnClass = getJavaClassFromPainlessType(whitelistMethod.painlessReturnTypeName);
+            painlessReturnClass = painlessTypeNameToPainlessType(whitelistMethod.painlessReturnTypeName);
         } catch (IllegalArgumentException iae) {
             throw new IllegalArgumentException("struct not defined for return type [" + whitelistMethod.painlessReturnTypeName + "] " +
                 "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
@@ -415,8 +429,8 @@ public class PainlessLookupBuilder {
                         "[" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
                 }
 
-                painlessMethod = methodCache.computeIfAbsent(
-                    buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes),
+                painlessMethod = painlessMethodCache.computeIfAbsent(
+                    new PainlessMethodCacheKey(ownerStruct.clazz, whitelistMethod.javaMethodName, painlessParametersTypes),
                     key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct.clazz, null, painlessReturnClass,
                         painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
                 ownerStruct.staticMethods.put(painlessMethodKey, painlessMethod);
@@ -441,8 +455,8 @@ public class PainlessLookupBuilder {
                         "[" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
                 }
 
-                painlessMethod = methodCache.computeIfAbsent(
-                    buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes),
+                painlessMethod = painlessMethodCache.computeIfAbsent(
+                    new PainlessMethodCacheKey(ownerStruct.clazz, whitelistMethod.javaMethodName, painlessParametersTypes),
                     key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct.clazz, javaAugmentedClass, painlessReturnClass,
                         painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
                 ownerStruct.methods.put(painlessMethodKey, painlessMethod);
@@ -457,14 +471,14 @@ public class PainlessLookupBuilder {
     }
 
     private void addField(String ownerStructName, WhitelistField whitelistField) {
-        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
                 "name [" + whitelistField.javaFieldName + "] and type " + whitelistField.painlessFieldTypeName);
         }
 
-        if (TYPE_NAME_PATTERN.matcher(whitelistField.javaFieldName).matches() == false) {
+        if (FIELD_NAME_PATTERN.matcher(whitelistField.javaFieldName).matches() == false) {
             throw new IllegalArgumentException("invalid field name " +
                 "[" + whitelistField.painlessFieldTypeName + "] for owner struct [" + ownerStructName + "].");
         }
@@ -481,7 +495,7 @@ public class PainlessLookupBuilder {
         Class<?> painlessFieldClass;
 
         try {
-            painlessFieldClass = getJavaClassFromPainlessType(whitelistField.painlessFieldTypeName);
+            painlessFieldClass = painlessTypeNameToPainlessType(whitelistField.painlessFieldTypeName);
         } catch (IllegalArgumentException iae) {
             throw new IllegalArgumentException("struct not defined for return type [" + whitelistField.painlessFieldTypeName + "] " +
                 "with owner struct [" + ownerStructName + "] and field with name [" + whitelistField.javaFieldName + "]", iae);
@@ -496,8 +510,8 @@ public class PainlessLookupBuilder {
             PainlessField painlessField = ownerStruct.staticMembers.get(whitelistField.javaFieldName);
 
             if (painlessField == null) {
-                painlessField = fieldCache.computeIfAbsent(
-                    buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()),
+                painlessField = painlessFieldCache.computeIfAbsent(
+                    new PainlessFieldCacheKey(ownerStruct.clazz, whitelistField.javaFieldName, painlessFieldClass),
                     key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
                         ownerStruct.clazz, painlessFieldClass, javaField.getModifiers(), null, null));
                 ownerStruct.staticMembers.put(whitelistField.javaFieldName, painlessField);
@@ -525,8 +539,8 @@ public class PainlessLookupBuilder {
             PainlessField painlessField = ownerStruct.members.get(whitelistField.javaFieldName);
 
             if (painlessField == null) {
-                painlessField = fieldCache.computeIfAbsent(
-                    buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()),
+                painlessField = painlessFieldCache.computeIfAbsent(
+                    new PainlessFieldCacheKey(ownerStruct.clazz, whitelistField.javaFieldName, painlessFieldClass),
                     key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
                         ownerStruct.clazz, painlessFieldClass, javaField.getModifiers(), javaMethodHandleGetter, javaMethodHandleSetter));
                 ownerStruct.members.put(whitelistField.javaFieldName, painlessField);
@@ -538,14 +552,15 @@ public class PainlessLookupBuilder {
     }
 
     private void copyStruct(String struct, List<String> children) {
-        final PainlessClassBuilder owner = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(struct));
+        final PainlessClassBuilder owner = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(struct));
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
         }
 
         for (int count = 0; count < children.size(); ++count) {
-            final PainlessClassBuilder child = javaClassesToPainlessClassBuilders.get(painlessTypesToJavaClasses.get(children.get(count)));
+            final PainlessClassBuilder child =
+                    javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(children.get(count)));
 
             if (child == null) {
                 throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
@@ -710,6 +725,122 @@ public class PainlessLookupBuilder {
     }
 
     public PainlessLookup build() {
+        String origin = "internal error";
+
+        try {
+            // first iteration collects all the Painless type names that
+            // are used for validation during the second iteration
+            for (Whitelist whitelist : whitelists) {
+                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
+                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
+                    PainlessClassBuilder painlessStruct =
+                        javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(painlessTypeName));
+
+                    if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
+                        throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
+                            "[" + painlessStruct.clazz.getName() + "] and [" + whitelistStruct.javaClassName + "]");
+                    }
+
+                    origin = whitelistStruct.origin;
+                    addPainlessClass(
+                            whitelist.javaClassLoader, whitelistStruct.javaClassName, whitelistStruct.onlyFQNJavaClassName == false);
+
+                    painlessStruct = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(painlessTypeName));
+                    javaClassesToPainlessClassBuilders.put(painlessStruct.clazz, painlessStruct);
+                }
+            }
+
+            // second iteration adds all the constructors, methods, and fields that will
+            // be available in Painless along with validating they exist and all their types have
+            // been white-listed during the first iteration
+            for (Whitelist whitelist : whitelists) {
+                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
+                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
+
+                    for (WhitelistConstructor whitelistConstructor : whitelistStruct.whitelistConstructors) {
+                        origin = whitelistConstructor.origin;
+                        addConstructor(painlessTypeName, whitelistConstructor);
+                    }
+
+                    for (WhitelistMethod whitelistMethod : whitelistStruct.whitelistMethods) {
+                        origin = whitelistMethod.origin;
+                        addMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod);
+                    }
+
+                    for (WhitelistField whitelistField : whitelistStruct.whitelistFields) {
+                        origin = whitelistField.origin;
+                        addField(painlessTypeName, whitelistField);
+                    }
+                }
+            }
+        } catch (Exception exception) {
+            throw new IllegalArgumentException("error loading whitelist(s) " + origin, exception);
+        }
+
+        // goes through each Painless struct and determines the inheritance list,
+        // and then adds all inherited types to the Painless struct's whitelist
+        for (Class<?> javaClass : javaClassesToPainlessClassBuilders.keySet()) {
+            PainlessClassBuilder painlessStruct = javaClassesToPainlessClassBuilders.get(javaClass);
+
+            List<String> painlessSuperStructs = new ArrayList<>();
+            Class<?> javaSuperClass = painlessStruct.clazz.getSuperclass();
+
+            Stack<Class<?>> javaInteraceLookups = new Stack<>();
+            javaInteraceLookups.push(painlessStruct.clazz);
+
+            // adds super classes to the inheritance list
+            if (javaSuperClass != null && javaSuperClass.isInterface() == false) {
+                while (javaSuperClass != null) {
+                    PainlessClassBuilder painlessSuperStruct = javaClassesToPainlessClassBuilders.get(javaSuperClass);
+
+                    if (painlessSuperStruct != null) {
+                        painlessSuperStructs.add(painlessSuperStruct.name);
+                    }
+
+                    javaInteraceLookups.push(javaSuperClass);
+                    javaSuperClass = javaSuperClass.getSuperclass();
+                }
+            }
+
+            // adds all super interfaces to the inheritance list
+            while (javaInteraceLookups.isEmpty() == false) {
+                Class<?> javaInterfaceLookup = javaInteraceLookups.pop();
+
+                for (Class<?> javaSuperInterface : javaInterfaceLookup.getInterfaces()) {
+                    PainlessClassBuilder painlessInterfaceStruct = javaClassesToPainlessClassBuilders.get(javaSuperInterface);
+
+                    if (painlessInterfaceStruct != null) {
+                        String painlessInterfaceStructName = painlessInterfaceStruct.name;
+
+                        if (painlessSuperStructs.contains(painlessInterfaceStructName) == false) {
+                            painlessSuperStructs.add(painlessInterfaceStructName);
+                        }
+
+                        for (Class<?> javaPushInterface : javaInterfaceLookup.getInterfaces()) {
+                            javaInteraceLookups.push(javaPushInterface);
+                        }
+                    }
+                }
+            }
+
+            // copies methods and fields from super structs to the parent struct
+            copyStruct(painlessStruct.name, painlessSuperStructs);
+
+            // copies methods and fields from Object into interface types
+            if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) {
+                PainlessClassBuilder painlessObjectStruct = javaClassesToPainlessClassBuilders.get(Object.class);
+
+                if (painlessObjectStruct != null) {
+                    copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name));
+                }
+            }
+        }
+
+        // precompute runtime classes
+        for (PainlessClassBuilder painlessStruct : javaClassesToPainlessClassBuilders.values()) {
+            addRuntimeClass(painlessStruct);
+        }
+
         Map<Class<?>, PainlessClass> javaClassesToPainlessClasses = new HashMap<>();
 
         // copy all structs to make them unmodifiable for outside users:
@@ -718,10 +849,6 @@ public class PainlessLookupBuilder {
             javaClassesToPainlessClasses.put(entry.getKey(), entry.getValue().build());
         }
 
-        return new PainlessLookup(painlessTypesToJavaClasses, javaClassesToPainlessClasses);
-    }
-
-    public Class<?> getJavaClassFromPainlessType(String painlessType) {
-        return PainlessLookupUtility.painlessTypeNameToPainlessType(painlessType, painlessTypesToJavaClasses);
+        return new PainlessLookup(painlessClassNamesToJavaClasses, javaClassesToPainlessClasses);
     }
 }

From 15ff3da653c5dd9eae7f7a654a9ee6a931d478b7 Mon Sep 17 00:00:00 2001
From: Julie Tibshirani <julie.tibshirani@elastic.co>
Date: Wed, 18 Jul 2018 09:33:09 -0700
Subject: [PATCH 088/260] Add support for field aliases. (#32172)

* Add basic support for field aliases in index mappings. (#31287)
* Allow for aliases when fetching stored fields. (#31411)
* Add tests around accessing field aliases in scripts. (#31417)
* Add documentation around field aliases. (#31538)
* Add validation for field alias mappings. (#31518)
* Return both concrete fields and aliases in DocumentFieldMappers#getMapper. (#31671)
* Make sure that field-level security is enforced when using field aliases. (#31807)
* Add more comprehensive tests for field aliases in queries + aggregations. (#31565)
* Remove the deprecated method DocumentFieldMappers#getFieldMapper. (#32148)
---
 docs/reference/indices/clearcache.asciidoc    |   3 +-
 docs/reference/mapping.asciidoc               |   6 +-
 docs/reference/mapping/types.asciidoc         |   4 +
 docs/reference/mapping/types/alias.asciidoc   | 101 ++++++++
 .../script/expression/ExpressionTests.java    |  67 +++++-
 .../percolator/PercolateQuery.java            |   4 +
 .../percolator/PercolateQueryBuilder.java     |  12 +-
 .../percolator/PercolatorFieldMapper.java     |  18 +-
 .../percolator/CandidateQueryTests.java       |   6 +-
 .../PercolateQueryBuilderTests.java           |  24 +-
 .../PercolatorFieldMapperTests.java           |   6 +-
 .../TransportGetFieldMappingsIndexAction.java |  18 +-
 ...TransportFieldCapabilitiesIndexAction.java |   4 +-
 .../org/elasticsearch/index/IndexWarmer.java  |  16 +-
 .../fieldvisitor/CustomFieldsVisitor.java     |  18 +-
 .../index/get/ShardGetService.java            |   4 +-
 .../index/mapper/DocumentFieldMappers.java    |  28 ++-
 .../index/mapper/DocumentMapper.java          |   5 +-
 .../index/mapper/DocumentParser.java          |  98 ++++----
 .../index/mapper/FieldAliasMapper.java        | 132 +++++++++++
 .../index/mapper/FieldMapper.java             |   5 +
 .../index/mapper/FieldTypeLookup.java         | 106 +++++++--
 .../elasticsearch/index/mapper/Mapper.java    |   5 +
 .../index/mapper/MapperMergeValidator.java    | 214 +++++++++++++++++
 .../index/mapper/MapperService.java           | 133 ++---------
 .../index/mapper/MapperUtils.java             |  17 +-
 .../index/mapper/ObjectMapper.java            |   5 +
 .../index/query/ExistsQueryBuilder.java       |  14 +-
 .../query/SpanMultiTermQueryBuilder.java      |  14 +-
 .../index/query/SpanNearQueryBuilder.java     |  29 ++-
 .../index/query/TermsSetQueryBuilder.java     |  21 +-
 .../index/search/QueryParserHelper.java       |  28 ++-
 .../elasticsearch/indices/IndicesModule.java  |   3 +
 .../SignificantTermsAggregatorFactory.java    |  15 +-
 .../SignificantTextAggregationBuilder.java    |   7 +-
 .../SignificantTextAggregatorFactory.java     |  30 ++-
 .../search/fetch/FetchPhase.java              | 124 +++++-----
 .../subphase/highlight/HighlightPhase.java    |   8 +-
 .../search/lookup/LeafFieldsLookup.java       |   2 +-
 .../search/suggest/SuggestionBuilder.java     |   3 +-
 .../completion/context/GeoContextMapping.java |   6 +-
 .../index/analysis/PreBuiltAnalyzerTests.java |  19 +-
 .../index/mapper/BinaryFieldMapperTests.java  |  27 ++-
 .../index/mapper/BooleanFieldMapperTests.java |   3 +-
 .../mapper/CompletionFieldMapperTests.java    |  98 +++++---
 .../index/mapper/CopyToMapperTests.java       |  12 +-
 .../index/mapper/DateFieldMapperTests.java    |   6 +-
 .../mapper/DocumentFieldMapperTests.java      |   8 +-
 .../mapper/DocumentMapperMergeTests.java      |  65 +++--
 .../index/mapper/DocumentParserTests.java     | 108 ++++++++-
 .../index/mapper/DoubleIndexingDocTests.java  |  19 +-
 .../index/mapper/DynamicMappingTests.java     |   8 +-
 .../index/mapper/DynamicTemplatesTests.java   |  21 +-
 .../index/mapper/FieldAliasMapperTests.java   | 167 +++++++++++++
 .../index/mapper/FieldTypeLookupTests.java    | 178 ++++++++++++--
 .../GenericStoreDynamicTemplateTests.java     |  20 +-
 .../mapper/GeoPointFieldMapperTests.java      |   6 +-
 .../mapper/GeoShapeFieldMapperTests.java      |  34 +--
 .../mapper/JavaMultiFieldMergeTests.java      |  98 ++++----
 .../mapper/MapperMergeValidatorTests.java     | 118 +++++++++
 .../index/mapper/MapperServiceTests.java      |  35 +++
 .../index/mapper/MultiFieldTests.java         |  65 ++---
 .../index/mapper/PathMapperTests.java         |   1 -
 .../mapper/PathMatchDynamicTemplateTests.java |  28 +--
 .../mapper/StoredNumericValuesTests.java      |   9 +-
 .../index/mapper/TextFieldMapperTests.java    |  53 +++--
 .../query/CommonTermsQueryBuilderTests.java   |  23 +-
 .../index/query/ExistsQueryBuilderTests.java  |   4 +-
 .../FieldMaskingSpanQueryBuilderTests.java    |   7 +-
 .../index/query/FuzzyQueryBuilderTests.java   |   8 +-
 .../GeoBoundingBoxQueryBuilderTests.java      |  11 +-
 .../query/GeoDistanceQueryBuilderTests.java   |   9 +-
 .../query/GeoPolygonQueryBuilderTests.java    |   3 +-
 .../MatchPhrasePrefixQueryBuilderTests.java   |   8 +-
 .../query/MatchPhraseQueryBuilderTests.java   |   8 +-
 .../index/query/MatchQueryBuilderTests.java   |  21 +-
 .../query/MoreLikeThisQueryBuilderTests.java  |   2 +-
 .../query/MultiMatchQueryBuilderTests.java    |   7 +-
 .../index/query/PrefixQueryBuilderTests.java  |   8 +-
 .../query/QueryStringQueryBuilderTests.java   |  16 +-
 .../index/query/RandomQueryBuilder.java       |  11 +-
 .../index/query/RangeQueryBuilderTests.java   |  31 +--
 .../index/query/RegexpQueryBuilderTests.java  |   6 +-
 .../query/SimpleQueryStringBuilderTests.java  |   3 +-
 .../query/SpanMultiTermQueryBuilderTests.java |  16 +-
 .../query/SpanTermQueryBuilderTests.java      |  18 +-
 .../index/query/TermQueryBuilderTests.java    |   7 +-
 .../index/query/TermsQueryBuilderTests.java   |  13 +-
 .../query/TermsSetQueryBuilderTests.java      |  25 +-
 .../query/WildcardQueryBuilderTests.java      |  25 +-
 .../index/similarity/SimilarityTests.java     |  92 ++++----
 .../mapping/SimpleGetFieldMappingsIT.java     |  40 +++-
 .../search/aggregations/bucket/RangeIT.java   |  83 +++++++
 .../aggregations/bucket/ReverseNestedIT.java  |  28 +++
 .../bucket/nested/NestedAggregatorTests.java  |  58 +++++
 .../nested/ReverseNestedAggregatorTests.java  |  76 ++++++
 .../SignificantTermsAggregatorTests.java      |  82 ++++++-
 .../SignificantTextAggregatorTests.java       | 133 ++++++++---
 .../bucket/terms/TermsAggregatorTests.java    |   8 +-
 .../search/aggregations/metrics/SumIT.java    |  92 +++++++-
 .../ScriptedMetricAggregatorTests.java        |   5 +-
 .../support/ValuesSourceConfigTests.java      |  23 ++
 .../highlight/HighlighterSearchIT.java        | 100 ++++++++
 .../search/fieldcaps/FieldCapabilitiesIT.java | 151 ++++++++++++
 .../search/fields/SearchFieldsIT.java         | 163 +++++++++++++
 .../search/geo/GeoPolygonIT.java              |  23 +-
 .../search/geo/GeoShapeQueryTests.java        |  42 +++-
 .../search/lookup/LeafDocLookupTests.java     |  75 ++++++
 .../search/lookup/LeafFieldsLookupTests.java  |  92 ++++++++
 .../search/morelikethis/MoreLikeThisIT.java   |  31 +++
 .../elasticsearch/search/query/ExistsIT.java  |  86 +++++++
 .../search/query/QueryStringIT.java           |  65 +++++
 .../search/query/SearchQueryIT.java           |  78 ++++++
 .../search/query/SimpleQueryStringIT.java     |  62 +++++
 .../search/sort/FieldSortIT.java              |  57 +++++
 .../AbstractSuggestionBuilderTestCase.java    |   8 +-
 .../suggest/CompletionSuggestSearchIT.java    |  28 ++-
 .../search/suggest/SuggestSearchIT.java       |  29 +++
 .../CategoryContextMappingTests.java          |  49 ++--
 .../CompletionSuggesterBuilderTests.java      |   3 +-
 .../completion/GeoContextMappingTests.java    |  58 ++---
 .../search/query/all-query-index.json         |   8 +
 .../aggregations/AggregatorTestCase.java      |  58 ++++-
 .../test/AbstractBuilderTestCase.java         |  60 +++--
 .../test/AbstractQueryTestCase.java           |  13 +-
 .../integration/FieldLevelSecurityTests.java  | 223 +++++++++++++++---
 126 files changed, 4090 insertions(+), 1051 deletions(-)
 create mode 100644 docs/reference/mapping/types/alias.asciidoc
 create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/MapperMergeValidatorTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java

diff --git a/docs/reference/indices/clearcache.asciidoc b/docs/reference/indices/clearcache.asciidoc
index 6a7240dc958..6b90f9d49cb 100644
--- a/docs/reference/indices/clearcache.asciidoc
+++ b/docs/reference/indices/clearcache.asciidoc
@@ -16,7 +16,8 @@ explicitly by setting `query`, `fielddata` or `request`.
 
 All caches relating to a specific field(s) can also be cleared by
 specifying `fields` parameter with a comma delimited list of the
-relevant fields.
+relevant fields. Note that the provided names must refer to concrete
+fields -- objects and field aliases are not supported.
 
 [float]
 === Multi Index
diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index 6f8f1b38d6f..31957344baf 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -124,8 +124,10 @@ fields to an existing index with the <<indices-put-mapping,PUT mapping API>>.
 
 Other than where documented, *existing field mappings cannot be
 updated*. Changing the mapping would mean invalidating already indexed
-documents.  Instead, you should create a new index with the correct mappings
-and <<docs-reindex,reindex>> your data into that index.
+documents. Instead, you should create a new index with the correct mappings
+and <<docs-reindex,reindex>> your data into that index. If you only wish
+to rename a field and not change its mappings, it may make sense to introduce
+an <<alias, `alias`>> field.
 
 [float]
 == Example mapping
diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index 9b71d7e7404..fbd8181d095 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -40,6 +40,8 @@ string::        <<text,`text`>> and <<keyword,`keyword`>>
 
 <<parent-join>>::   Defines parent/child relation for documents within the same index
 
+<<alias>>::         Defines an alias to an existing field.
+
 <<feature>>::       Record numeric features to boost hits at query time.
 
 <<feature-vector>>:: Record numeric feature vectors to boost hits at query time.
@@ -58,6 +60,8 @@ the <<analysis-standard-analyzer,`standard` analyzer>>, the
 This is the purpose of _multi-fields_.  Most datatypes support multi-fields
 via the <<multi-fields>> parameter.
 
+include::types/alias.asciidoc[]
+
 include::types/array.asciidoc[]
 
 include::types/binary.asciidoc[]
diff --git a/docs/reference/mapping/types/alias.asciidoc b/docs/reference/mapping/types/alias.asciidoc
new file mode 100644
index 00000000000..d2b5ccdce8a
--- /dev/null
+++ b/docs/reference/mapping/types/alias.asciidoc
@@ -0,0 +1,101 @@
+[[alias]]
+=== Alias datatype
+
+An `alias` mapping defines an alternate name for a field in the  index.
+The alias can be used in place of the target field in <<search, search>> requests,
+and selected other APIs like <<search-field-caps, field capabilities>>.
+
+[source,js]
+--------------------------------
+PUT trips
+{
+  "mappings": {
+    "_doc": {
+      "properties": {
+        "distance": {
+          "type": "long"
+        },
+        "route_length_miles": {
+          "type": "alias",
+          "path": "distance" // <1>
+        },
+        "transit_mode": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+
+GET _search
+{
+  "query": {
+    "range" : {
+      "route_length_miles" : {
+        "gte" : 39
+      }
+    }
+  }
+}
+--------------------------------
+// CONSOLE
+
+<1> The path to the target field. Note that this must be the full path, including any parent
+objects (e.g. `object1.object2.field`).
+
+Almost all components of the search request accept field aliases. In particular, aliases can be
+used in queries, aggregations, and sort fields, as well as when requesting `docvalue_fields`,
+`stored_fields`, suggestions, and highlights. Scripts also support aliases when accessing
+field values. Please see the section on <<unsupported-apis, unsupported APIs>> for exceptions.
+
+In some parts of the search request and when requesting field capabilities, field wildcard patterns can be
+provided. In these cases, the wildcard pattern will match field aliases in addition to concrete fields:
+
+[source,js]
+--------------------------------
+GET trips/_field_caps?fields=route_*,transit_mode
+--------------------------------
+// CONSOLE
+// TEST[continued]
+
+[[alias-targets]]
+==== Alias targets
+
+There are a few restrictions on the target of an alias:
+
+  * The target must be a concrete field, and not an object or another field alias.
+  * The target field must exist at the time the alias is created.
+  * If nested objects are defined, a field alias must have the same nested scope as its target.
+
+Additionally, a field alias can only have one target. This means that it is not possible to use a
+field alias to query over multiple target fields in a single clause.
+
+[[unsupported-apis]]
+==== Unsupported APIs
+
+Writes to field aliases are not supported: attempting to use an alias in an index or update request
+will result in a failure. Likewise, aliases cannot be used as the target of `copy_to`.
+
+Because alias names are not present in the document source, aliases cannot be used when performing
+source filtering. For example, the following request will return an empty result for `_source`:
+
+[source,js]
+--------------------------------
+GET /_search
+{
+  "query" : {
+    "match_all": {}
+  },
+  "_source": "route_length_miles"
+}
+--------------------------------
+// CONSOLE
+// TEST[continued]
+
+Currently only the search and field capabilities APIs will accept and resolve field aliases.
+Other APIs that accept field names, such as <<docs-termvectors, term vectors>>, cannot be used
+with field aliases.
+
+Finally, some queries, such as `terms`, `geo_shape`, and `more_like_this`, allow for fetching query
+information from an indexed document. Because field aliases aren't supported when fetching documents,
+the part of the query that specifies the lookup path cannot refer to a field by its alias.
\ No newline at end of file
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
index 591cc9ce477..33e6239002e 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
@@ -20,27 +20,52 @@
 package org.elasticsearch.script.expression;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.text.ParseException;
 import java.util.Collections;
 
-public class ExpressionTests extends ESSingleNodeTestCase {
-    ExpressionScriptEngine service;
-    SearchLookup lookup;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ExpressionTests extends ESTestCase {
+    private ExpressionScriptEngine service;
+    private SearchLookup lookup;
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        IndexService index = createIndex("test", Settings.EMPTY, "type", "d", "type=double");
+
+        NumberFieldType fieldType = new NumberFieldType(NumberType.DOUBLE);
+        MapperService mapperService = mock(MapperService.class);
+        when(mapperService.fullName("field")).thenReturn(fieldType);
+        when(mapperService.fullName("alias")).thenReturn(fieldType);
+
+        SortedNumericDoubleValues doubleValues = mock(SortedNumericDoubleValues.class);
+        when(doubleValues.advanceExact(anyInt())).thenReturn(true);
+        when(doubleValues.nextValue()).thenReturn(2.718);
+
+        AtomicNumericFieldData atomicFieldData = mock(AtomicNumericFieldData.class);
+        when(atomicFieldData.getDoubleValues()).thenReturn(doubleValues);
+
+        IndexNumericFieldData fieldData = mock(IndexNumericFieldData.class);
+        when(fieldData.getFieldName()).thenReturn("field");
+        when(fieldData.load(anyObject())).thenReturn(atomicFieldData);
+
         service = new ExpressionScriptEngine(Settings.EMPTY);
-        QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null);
-        lookup = new SearchLookup(index.mapperService(), shardContext::getForField, null);
+        lookup = new SearchLookup(mapperService, ignored -> fieldData, null);
     }
 
     private SearchScript.LeafFactory compile(String expression) {
@@ -50,22 +75,38 @@ public class ExpressionTests extends ESSingleNodeTestCase {
 
     public void testNeedsScores() {
         assertFalse(compile("1.2").needs_score());
-        assertFalse(compile("doc['d'].value").needs_score());
+        assertFalse(compile("doc['field'].value").needs_score());
         assertTrue(compile("1/_score").needs_score());
-        assertTrue(compile("doc['d'].value * _score").needs_score());
+        assertTrue(compile("doc['field'].value * _score").needs_score());
     }
 
     public void testCompileError() {
         ScriptException e = expectThrows(ScriptException.class, () -> {
-            compile("doc['d'].value * *@#)(@$*@#$ + 4");
+            compile("doc['field'].value * *@#)(@$*@#$ + 4");
         });
         assertTrue(e.getCause() instanceof ParseException);
     }
 
     public void testLinkError() {
         ScriptException e = expectThrows(ScriptException.class, () -> {
-            compile("doc['e'].value * 5");
+            compile("doc['nonexistent'].value * 5");
         });
         assertTrue(e.getCause() instanceof ParseException);
     }
+
+    public void testFieldAccess() throws IOException {
+        SearchScript script = compile("doc['field'].value").newInstance(null);
+        script.setDocument(1);
+
+        double result = script.runAsDouble();
+        assertEquals(2.718, result, 0.0);
+    }
+
+    public void testFieldAccessWithFieldAlias() throws IOException {
+        SearchScript script = compile("doc['alias'].value").newInstance(null);
+        script.setDocument(1);
+
+        double result = script.runAsDouble();
+        assertEquals(2.718, result, 0.0);
+    }
 }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index f24a9710d29..5bbf998883e 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -194,6 +194,10 @@ final class PercolateQuery extends Query implements Accountable {
         return candidateMatchesQuery;
     }
 
+    Query getVerifiedMatchesQuery() {
+        return verifiedMatchesQuery;
+    }
+
     // Comparing identity here to avoid being cached
     // Note that in theory if the same instance gets used multiple times it could still get cached,
     // however since we create a new query instance each time we this query this shouldn't happen and thus
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index c1063795193..e28fbead29a 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -618,13 +618,13 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
             docSearcher.setQueryCache(null);
         }
 
-        PercolatorFieldMapper percolatorFieldMapper = (PercolatorFieldMapper) docMapper.mappers().getMapper(field);
-        boolean mapUnmappedFieldsAsString = percolatorFieldMapper.isMapUnmappedFieldAsText();
-        QueryShardContext percolateShardContext = wrap(context);
-
-        String name = this.name != null ? this.name : field;
         PercolatorFieldMapper.FieldType pft = (PercolatorFieldMapper.FieldType) fieldType;
-        PercolateQuery.QueryStore queryStore = createStore(pft.queryBuilderField, percolateShardContext, mapUnmappedFieldsAsString);
+        String name = this.name != null ? this.name : pft.name();
+        QueryShardContext percolateShardContext = wrap(context);
+        PercolateQuery.QueryStore queryStore = createStore(pft.queryBuilderField,
+            percolateShardContext,
+            pft.mapUnmappedFieldsAsText);
+
         return pft.percolateQuery(name, queryStore, documents, docSearcher, context.indexVersionCreated());
     }
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index 103679f5328..414d1471e5c 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -136,6 +136,8 @@ public class PercolatorFieldMapper extends FieldMapper {
             fieldType.rangeField = rangeFieldMapper.fieldType();
             NumberFieldMapper minimumShouldMatchFieldMapper = createMinimumShouldMatchField(context);
             fieldType.minimumShouldMatchField = minimumShouldMatchFieldMapper.fieldType();
+            fieldType.mapUnmappedFieldsAsText = getMapUnmappedFieldAsText(context.indexSettings());
+
             context.path().remove();
             setupFieldType(context);
             return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(),
@@ -143,6 +145,10 @@ public class PercolatorFieldMapper extends FieldMapper {
                     extractionResultField, queryBuilderField, rangeFieldMapper, minimumShouldMatchFieldMapper);
         }
 
+        private static boolean getMapUnmappedFieldAsText(Settings indexSettings) {
+            return INDEX_MAP_UNMAPPED_FIELDS_AS_TEXT_SETTING.get(indexSettings);
+        }
+
         static KeywordFieldMapper createExtractQueryFieldBuilder(String name, BuilderContext context) {
             KeywordFieldMapper.Builder queryMetaDataFieldBuilder = new KeywordFieldMapper.Builder(name);
             queryMetaDataFieldBuilder.docValues(false);
@@ -195,6 +201,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         MappedFieldType minimumShouldMatchField;
 
         RangeFieldMapper.RangeFieldType rangeField;
+        boolean mapUnmappedFieldsAsText;
 
         FieldType() {
             setIndexOptions(IndexOptions.NONE);
@@ -209,6 +216,7 @@ public class PercolatorFieldMapper extends FieldMapper {
             queryBuilderField = ref.queryBuilderField;
             rangeField = ref.rangeField;
             minimumShouldMatchField = ref.minimumShouldMatchField;
+            mapUnmappedFieldsAsText = ref.mapUnmappedFieldsAsText;
         }
 
         @Override
@@ -327,7 +335,6 @@ public class PercolatorFieldMapper extends FieldMapper {
 
     }
 
-    private final boolean mapUnmappedFieldAsText;
     private final Supplier<QueryShardContext> queryShardContext;
     private KeywordFieldMapper queryTermsField;
     private KeywordFieldMapper extractionResultField;
@@ -348,14 +355,9 @@ public class PercolatorFieldMapper extends FieldMapper {
         this.extractionResultField = extractionResultField;
         this.queryBuilderField = queryBuilderField;
         this.minimumShouldMatchFieldMapper = minimumShouldMatchFieldMapper;
-        this.mapUnmappedFieldAsText = getMapUnmappedFieldAsText(indexSettings);
         this.rangeFieldMapper = rangeFieldMapper;
     }
 
-    private static boolean getMapUnmappedFieldAsText(Settings indexSettings) {
-        return INDEX_MAP_UNMAPPED_FIELDS_AS_TEXT_SETTING.get(indexSettings);
-    }
-
     @Override
     public FieldMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
         PercolatorFieldMapper updated = (PercolatorFieldMapper) super.updateFieldType(fullNameToFieldType);
@@ -402,7 +404,7 @@ public class PercolatorFieldMapper extends FieldMapper {
 
         Version indexVersion = context.mapperService().getIndexSettings().getIndexVersionCreated();
         createQueryBuilderField(indexVersion, queryBuilderField, queryBuilder, context);
-        Query query = toQuery(queryShardContext, mapUnmappedFieldAsText, queryBuilder);
+        Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilder);
         processQuery(query, context);
         return null;
     }
@@ -522,7 +524,7 @@ public class PercolatorFieldMapper extends FieldMapper {
     }
 
     boolean isMapUnmappedFieldAsText() {
-        return mapUnmappedFieldAsText;
+        return ((FieldType) fieldType).mapUnmappedFieldsAsText;
     }
 
     /**
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
index 106358b6cf0..e6d637aabb1 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
@@ -194,8 +194,7 @@ public class CandidateQueryTests extends ESSingleNodeTestCase {
         }
         Collections.sort(intValues);
 
-        MappedFieldType intFieldType = mapperService.documentMapper("type").mappers()
-            .getMapper("int_field").fieldType();
+        MappedFieldType intFieldType = mapperService.fullName("int_field");
 
         List<Supplier<Query>> queryFunctions = new ArrayList<>();
         queryFunctions.add(MatchNoDocsQuery::new);
@@ -327,8 +326,7 @@ public class CandidateQueryTests extends ESSingleNodeTestCase {
         stringValues.add("value2");
         stringValues.add("value3");
 
-        MappedFieldType intFieldType = mapperService.documentMapper("type").mappers()
-            .getMapper("int_field").fieldType();
+        MappedFieldType intFieldType = mapperService.fullName("int_field");
         List<int[]> ranges = new ArrayList<>();
         ranges.add(new int[]{-5, 5});
         ranges.add(new int[]{0, 10});
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index 3b3ff4ed15c..e7163edef94 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -75,7 +75,8 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
         PercolateQueryBuilder.DOCUMENTS_FIELD.getPreferredName()
     };
 
-    private static String queryField;
+    private static String queryField = "field";
+    private static String aliasField = "alias";
     private static String docType;
 
     private String indexedDocumentIndex;
@@ -96,9 +97,11 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
     @Override
     protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
         queryField = randomAlphaOfLength(4);
+        aliasField = randomAlphaOfLength(4);
+
         String docType = "_doc";
         mapperService.merge(docType, new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef(docType,
-                queryField, "type=percolator"
+                queryField, "type=percolator", aliasField, "type=alias,path=" + queryField
         ))), MapperService.MergeReason.MAPPING_UPDATE);
         mapperService.merge(docType, new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef(docType,
                 STRING_FIELD_NAME, "type=text"
@@ -355,4 +358,21 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
         builder = rewriteAndFetch(builder, createShardContext());
         builder.writeTo(new BytesStreamOutput(10));
     }
+
+    public void testFieldAlias() throws IOException {
+        QueryShardContext shardContext = createShardContext();
+
+        PercolateQueryBuilder builder = doCreateTestQueryBuilder(false);
+        QueryBuilder rewrittenBuilder = rewriteAndFetch(builder, shardContext);
+        PercolateQuery query = (PercolateQuery) rewrittenBuilder.toQuery(shardContext);
+
+        PercolateQueryBuilder aliasBuilder = new PercolateQueryBuilder(aliasField,
+            builder.getDocuments(),
+            builder.getXContentType());
+        QueryBuilder rewrittenAliasBuilder = rewriteAndFetch(aliasBuilder, shardContext);
+        PercolateQuery aliasQuery = (PercolateQuery) rewrittenAliasBuilder.toQuery(shardContext);
+
+        assertEquals(query.getCandidateMatchesQuery(), aliasQuery.getCandidateMatchesQuery());
+        assertEquals(query.getVerifiedMatchesQuery(), aliasQuery.getVerifiedMatchesQuery());
+    }
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
index 5b5ac41d25f..ecff48b344c 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
@@ -224,10 +224,10 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
     public void testExtractRanges() throws Exception {
         addQueryFieldMappings();
         BooleanQuery.Builder bq = new BooleanQuery.Builder();
-        Query rangeQuery1 = mapperService.documentMapper("doc").mappers().getMapper("number_field1").fieldType()
+        Query rangeQuery1 = mapperService.fullName("number_field1")
             .rangeQuery(10, 20, true, true, null, null, null, null);
         bq.add(rangeQuery1, Occur.MUST);
-        Query rangeQuery2 = mapperService.documentMapper("doc").mappers().getMapper("number_field1").fieldType()
+        Query rangeQuery2 = mapperService.fullName("number_field1")
             .rangeQuery(15, 20, true, true, null, null, null, null);
         bq.add(rangeQuery2, Occur.MUST);
 
@@ -255,7 +255,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         // Range queries on different fields:
         bq = new BooleanQuery.Builder();
         bq.add(rangeQuery1, Occur.MUST);
-        rangeQuery2 = mapperService.documentMapper("doc").mappers().getMapper("number_field2").fieldType()
+        rangeQuery2 = mapperService.fullName("number_field2")
             .rangeQuery(15, 20, true, true, null, null, null, null);
         bq.add(rangeQuery2, Occur.MUST);
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
index 07ae513351e..f9fc5880bbb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
@@ -39,7 +39,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentFieldMappers;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.TypeMissingException;
@@ -174,19 +174,19 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
         final DocumentFieldMappers allFieldMappers = documentMapper.mappers();
         for (String field : request.fields()) {
             if (Regex.isMatchAllPattern(field)) {
-                for (FieldMapper fieldMapper : allFieldMappers) {
-                    addFieldMapper(fieldPredicate, fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
+                for (Mapper fieldMapper : allFieldMappers) {
+                    addFieldMapper(fieldPredicate, fieldMapper.name(), fieldMapper, fieldMappings, request.includeDefaults());
                 }
             } else if (Regex.isSimpleMatchPattern(field)) {
-                for (FieldMapper fieldMapper : allFieldMappers) {
-                    if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
-                        addFieldMapper(fieldPredicate,  fieldMapper.fieldType().name(),
+                for (Mapper fieldMapper : allFieldMappers) {
+                    if (Regex.simpleMatch(field, fieldMapper.name())) {
+                        addFieldMapper(fieldPredicate,  fieldMapper.name(),
                                 fieldMapper, fieldMappings, request.includeDefaults());
                     }
                 }
             } else {
                 // not a pattern
-                FieldMapper fieldMapper = allFieldMappers.getMapper(field);
+                Mapper fieldMapper = allFieldMappers.getMapper(field);
                 if (fieldMapper != null) {
                     addFieldMapper(fieldPredicate, field, fieldMapper, fieldMappings, request.includeDefaults());
                 } else if (request.probablySingleFieldRequest()) {
@@ -198,7 +198,7 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
     }
 
     private static void addFieldMapper(Predicate<String> fieldPredicate,
-                                       String field, FieldMapper fieldMapper, Map<String, FieldMappingMetaData> fieldMappings,
+                                       String field, Mapper fieldMapper, Map<String, FieldMappingMetaData> fieldMappings,
                                        boolean includeDefaults) {
         if (fieldMappings.containsKey(field)) {
             return;
@@ -207,7 +207,7 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
             try {
                 BytesReference bytes = XContentHelper.toXContent(fieldMapper, XContentType.JSON,
                         includeDefaults ? includeDefaultsParams : ToXContent.EMPTY_PARAMS, false);
-                fieldMappings.put(field, new FieldMappingMetaData(fieldMapper.fieldType().name(), bytes));
+                fieldMappings.put(field, new FieldMappingMetaData(fieldMapper.name(), bytes));
             } catch (IOException e) {
                 throw new ElasticsearchException("failed to serialize XContent of field [" + field + "]", e);
             }
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
index f1a1dc45140..18f33ab397f 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
@@ -83,8 +83,8 @@ public class TransportFieldCapabilitiesIndexAction extends TransportSingleShardA
         for (String field : fieldNames) {
             MappedFieldType ft = mapperService.fullName(field);
             if (ft != null) {
-                FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
-                if (indicesService.isMetaDataField(field) || fieldPredicate.test(field)) {
+                if (indicesService.isMetaDataField(field) || fieldPredicate.test(ft.name())) {
+                    FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
                     responseMap.put(field, fieldCap);
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java
index e06dc5d2e81..98716e9545d 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java
@@ -27,8 +27,6 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.IndexShard;
@@ -121,16 +119,12 @@ public final class IndexWarmer extends AbstractComponent {
         public TerminationHandle warmReader(final IndexShard indexShard, final Engine.Searcher searcher) {
             final MapperService mapperService = indexShard.mapperService();
             final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
-            DocumentMapper docMapper = mapperService.documentMapper();
-            if (docMapper != null) {
-                for (FieldMapper fieldMapper : docMapper.mappers()) {
-                    final MappedFieldType fieldType = fieldMapper.fieldType();
-                    final String indexName = fieldType.name();
-                    if (fieldType.eagerGlobalOrdinals() == false) {
-                        continue;
-                    }
-                    warmUpGlobalOrdinals.put(indexName, fieldType);
+            for (MappedFieldType fieldType : mapperService.fieldTypes()) {
+                final String indexName = fieldType.name();
+                if (fieldType.eagerGlobalOrdinals() == false) {
+                    continue;
                 }
+                warmUpGlobalOrdinals.put(indexName, fieldType);
             }
             final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
             for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java
index bd1fd69eb74..c5d5a688ed9 100644
--- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java
+++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java
@@ -19,11 +19,8 @@
 package org.elasticsearch.index.fieldvisitor;
 
 import org.apache.lucene.index.FieldInfo;
-import org.elasticsearch.common.regex.Regex;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /**
@@ -35,16 +32,10 @@ import java.util.Set;
 public class CustomFieldsVisitor extends FieldsVisitor {
 
     private final Set<String> fields;
-    private final List<String> patterns;
-
-    public CustomFieldsVisitor(Set<String> fields, List<String> patterns, boolean loadSource) {
-        super(loadSource);
-        this.fields = fields;
-        this.patterns = patterns;
-    }
 
     public CustomFieldsVisitor(Set<String> fields, boolean loadSource) {
-        this(fields, Collections.emptyList(), loadSource);
+        super(loadSource);
+        this.fields = fields;
     }
 
     @Override
@@ -55,11 +46,6 @@ public class CustomFieldsVisitor extends FieldsVisitor {
         if (fields.contains(fieldInfo.name)) {
             return Status.YES;
         }
-        for (String pattern : patterns) {
-            if (Regex.simpleMatch(pattern, fieldInfo.name)) {
-                return Status.YES;
-            }
-        }
         return Status.NO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java
index a759f6a6767..5c5554cddcf 100644
--- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java
+++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java
@@ -39,7 +39,7 @@ import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
 import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -202,7 +202,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
 
         if (gFields != null && gFields.length > 0) {
             for (String field : gFields) {
-                FieldMapper fieldMapper = docMapper.mappers().getMapper(field);
+                Mapper fieldMapper = docMapper.mappers().getMapper(field);
                 if (fieldMapper == null) {
                     if (docMapper.objectMappers().get(field) != null) {
                         // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
index 9193ca209ba..f70c0038464 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
@@ -28,10 +28,10 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 
-public final class DocumentFieldMappers implements Iterable<FieldMapper> {
+public final class DocumentFieldMappers implements Iterable<Mapper> {
 
     /** Full field name to mapper */
-    private final Map<String, FieldMapper> fieldMappers;
+    private final Map<String, Mapper> fieldMappers;
 
     private final FieldNameAnalyzer indexAnalyzer;
     private final FieldNameAnalyzer searchAnalyzer;
@@ -44,8 +44,12 @@ public final class DocumentFieldMappers implements Iterable<FieldMapper> {
         analyzers.put(key, value);
     }
 
-    public DocumentFieldMappers(Collection<FieldMapper> mappers, Analyzer defaultIndex, Analyzer defaultSearch, Analyzer defaultSearchQuote) {
-        Map<String, FieldMapper> fieldMappers = new HashMap<>();
+    public DocumentFieldMappers(Collection<FieldMapper> mappers,
+                                Collection<FieldAliasMapper> aliasMappers,
+                                Analyzer defaultIndex,
+                                Analyzer defaultSearch,
+                                Analyzer defaultSearchQuote) {
+        Map<String, Mapper> fieldMappers = new HashMap<>();
         Map<String, Analyzer> indexAnalyzers = new HashMap<>();
         Map<String, Analyzer> searchAnalyzers = new HashMap<>();
         Map<String, Analyzer> searchQuoteAnalyzers = new HashMap<>();
@@ -56,14 +60,24 @@ public final class DocumentFieldMappers implements Iterable<FieldMapper> {
             put(searchAnalyzers, fieldType.name(), fieldType.searchAnalyzer(), defaultSearch);
             put(searchQuoteAnalyzers, fieldType.name(), fieldType.searchQuoteAnalyzer(), defaultSearchQuote);
         }
+
+        for (FieldAliasMapper aliasMapper : aliasMappers) {
+            fieldMappers.put(aliasMapper.name(), aliasMapper);
+        }
+
         this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
         this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);
         this.searchAnalyzer = new FieldNameAnalyzer(searchAnalyzers);
         this.searchQuoteAnalyzer = new FieldNameAnalyzer(searchQuoteAnalyzers);
     }
 
-    /** Returns the mapper for the given field */
-    public FieldMapper getMapper(String field) {
+    /**
+     * Returns the leaf mapper associated with this field name. Note that the returned mapper
+     * could be either a concrete {@link FieldMapper}, or a {@link FieldAliasMapper}.
+     *
+     * To access a field's type information, {@link MapperService#fullName} should be used instead.
+     */
+    public Mapper getMapper(String field) {
         return fieldMappers.get(field);
     }
 
@@ -87,7 +101,7 @@ public final class DocumentFieldMappers implements Iterable<FieldMapper> {
         return this.searchQuoteAnalyzer;
     }
 
-    public Iterator<FieldMapper> iterator() {
+    public Iterator<Mapper> iterator() {
         return fieldMappers.values().iterator();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index 3ec0f544355..87942260742 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -133,15 +133,18 @@ public class DocumentMapper implements ToXContentFragment {
         // collect all the mappers for this type
         List<ObjectMapper> newObjectMappers = new ArrayList<>();
         List<FieldMapper> newFieldMappers = new ArrayList<>();
+        List<FieldAliasMapper> newFieldAliasMappers = new ArrayList<>();
         for (MetadataFieldMapper metadataMapper : this.mapping.metadataMappers) {
             if (metadataMapper instanceof FieldMapper) {
                 newFieldMappers.add(metadataMapper);
             }
         }
-        MapperUtils.collect(this.mapping.root, newObjectMappers, newFieldMappers);
+        MapperUtils.collect(this.mapping.root,
+            newObjectMappers, newFieldMappers, newFieldAliasMappers);
 
         final IndexAnalyzers indexAnalyzers = mapperService.getIndexAnalyzers();
         this.fieldMappers = new DocumentFieldMappers(newFieldMappers,
+                newFieldAliasMappers,
                 indexAnalyzers.getDefaultIndexAnalyzer(),
                 indexAnalyzers.getDefaultSearchAnalyzer(),
                 indexAnalyzers.getDefaultSearchQuoteAnalyzer());
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 61ff4a4ff3d..0fd156c0905 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -459,13 +459,18 @@ final class DocumentParser {
     private static void parseObjectOrField(ParseContext context, Mapper mapper) throws IOException {
         if (mapper instanceof ObjectMapper) {
             parseObjectOrNested(context, (ObjectMapper) mapper);
-        } else {
-            FieldMapper fieldMapper = (FieldMapper)mapper;
+        } else if (mapper instanceof FieldMapper) {
+            FieldMapper fieldMapper = (FieldMapper) mapper;
             Mapper update = fieldMapper.parse(context);
             if (update != null) {
                 context.addDynamicMapper(update);
             }
             parseCopyFields(context, fieldMapper.copyTo().copyToFields());
+        } else if (mapper instanceof FieldAliasMapper) {
+            throw new IllegalArgumentException("Cannot write to a field alias [" + mapper.name() + "].");
+        } else {
+            throw new IllegalStateException("The provided mapper [" + mapper.name() + "] has an unrecognized type [" +
+                mapper.getClass().getSimpleName() + "].");
         }
     }
 
@@ -827,9 +832,16 @@ final class DocumentParser {
 
     /** Creates an copy of the current field with given field name and boost */
     private static void parseCopy(String field, ParseContext context) throws IOException {
-        FieldMapper fieldMapper = context.docMapper().mappers().getMapper(field);
-        if (fieldMapper != null) {
-            fieldMapper.parse(context);
+        Mapper mapper = context.docMapper().mappers().getMapper(field);
+        if (mapper != null) {
+            if (mapper instanceof FieldMapper) {
+                ((FieldMapper) mapper).parse(context);
+            } else if (mapper instanceof FieldAliasMapper) {
+                throw new IllegalArgumentException("Cannot copy to a field alias [" + mapper.name() + "].");
+            } else {
+                throw new IllegalStateException("The provided mapper [" + mapper.name() +
+                    "] has an unrecognized type [" + mapper.getClass().getSimpleName() + "].");
+            }
         } else {
             // The path of the dest field might be completely different from the current one so we need to reset it
             context = context.overridePath(new ContentPath(0));
@@ -837,8 +849,8 @@ final class DocumentParser {
             final String[] paths = splitAndValidatePath(field);
             final String fieldName = paths[paths.length-1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, null);
-            ObjectMapper mapper = parentMapperTuple.v2();
-            parseDynamicValue(context, mapper, fieldName, context.parser().currentToken());
+            ObjectMapper objectMapper = parentMapperTuple.v2();
+            parseDynamicValue(context, objectMapper, fieldName, context.parser().currentToken());
             for (int i = 0; i < parentMapperTuple.v1(); i++) {
                 context.path().remove();
             }
@@ -849,46 +861,46 @@ final class DocumentParser {
             ObjectMapper currentParent) {
         ObjectMapper mapper = currentParent == null ? context.root() : currentParent;
         int pathsAdded = 0;
-            ObjectMapper parent = mapper;
-            for (int i = 0; i < paths.length-1; i++) {
-            String currentPath = context.path().pathAsText(paths[i]);
-            FieldMapper existingFieldMapper = context.docMapper().mappers().getMapper(currentPath);
-            if (existingFieldMapper != null) {
-                throw new MapperParsingException(
-                        "Could not dynamically add mapping for field [{}]. Existing mapping for [{}] must be of type object but found [{}].",
-                        null, String.join(".", paths), currentPath, existingFieldMapper.fieldType.typeName());
-            }
-            mapper = context.docMapper().objectMappers().get(currentPath);
-                if (mapper == null) {
-                    // One mapping is missing, check if we are allowed to create a dynamic one.
-                    ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context);
+        ObjectMapper parent = mapper;
+        for (int i = 0; i < paths.length-1; i++) {
+        String currentPath = context.path().pathAsText(paths[i]);
+        Mapper existingFieldMapper = context.docMapper().mappers().getMapper(currentPath);
+        if (existingFieldMapper != null) {
+            throw new MapperParsingException(
+                    "Could not dynamically add mapping for field [{}]. Existing mapping for [{}] must be of type object but found [{}].",
+                    null, String.join(".", paths), currentPath, existingFieldMapper.typeName());
+        }
+        mapper = context.docMapper().objectMappers().get(currentPath);
+            if (mapper == null) {
+                // One mapping is missing, check if we are allowed to create a dynamic one.
+                ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context);
 
-                    switch (dynamic) {
-                        case STRICT:
-                            throw new StrictDynamicMappingException(parent.fullPath(), paths[i]);
-                        case TRUE:
-                            Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT);
-                            if (builder == null) {
-                                builder = new ObjectMapper.Builder(paths[i]).enabled(true);
-                            }
-                            Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
-                            mapper = (ObjectMapper) builder.build(builderContext);
-                            if (mapper.nested() != ObjectMapper.Nested.NO) {
-                                throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().pathAsText(paths[i])
-                                        + "]) through `copy_to` or dots in field names");
-                            }
-                            context.addDynamicMapper(mapper);
-                            break;
-                        case FALSE:
-                           // Should not dynamically create any more mappers so return the last mapper
-                        return new Tuple<>(pathsAdded, parent);
+                switch (dynamic) {
+                    case STRICT:
+                        throw new StrictDynamicMappingException(parent.fullPath(), paths[i]);
+                    case TRUE:
+                        Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT);
+                        if (builder == null) {
+                            builder = new ObjectMapper.Builder(paths[i]).enabled(true);
+                        }
+                        Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
+                        mapper = (ObjectMapper) builder.build(builderContext);
+                        if (mapper.nested() != ObjectMapper.Nested.NO) {
+                            throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().pathAsText(paths[i])
+                                    + "]) through `copy_to` or dots in field names");
+                        }
+                        context.addDynamicMapper(mapper);
+                        break;
+                    case FALSE:
+                       // Should not dynamically create any more mappers so return the last mapper
+                    return new Tuple<>(pathsAdded, parent);
 
-                    }
                 }
-                context.path().add(paths[i]);
-                pathsAdded++;
-                parent = mapper;
             }
+            context.path().add(paths[i]);
+            pathsAdded++;
+            parent = mapper;
+        }
         return new Tuple<>(pathsAdded, mapper);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java
new file mode 100644
index 00000000000..8d87b4f73ec
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * A mapper for field aliases.
+ *
+ * A field alias has no concrete field mappings of its own, but instead points to another field by
+ * its path. Once defined, an alias can be used in place of the concrete field name in search requests.
+ */
+public final class FieldAliasMapper extends Mapper {
+    public static final String CONTENT_TYPE = "alias";
+
+    public static class Names {
+        public static final String PATH = "path";
+    }
+
+    private final String name;
+    private final String path;
+
+    public FieldAliasMapper(String simpleName,
+                            String name,
+                            String path) {
+        super(simpleName);
+        this.name = name;
+        this.path = path;
+    }
+
+    @Override
+    public String name() {
+        return name;
+    }
+
+    @Override
+    public String typeName() {
+        return CONTENT_TYPE;
+    }
+
+    public String path() {
+        return path;
+    }
+
+    @Override
+    public Mapper merge(Mapper mergeWith) {
+        if (!(mergeWith instanceof FieldAliasMapper)) {
+            throw new IllegalArgumentException("Cannot merge a field alias mapping ["
+                + name() + "] with a mapping that is not for a field alias.");
+        }
+        return mergeWith;
+    }
+
+    @Override
+    public Mapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
+        return this;
+    }
+
+    @Override
+    public Iterator<Mapper> iterator() {
+        return Collections.emptyIterator();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject(simpleName())
+            .field("type", CONTENT_TYPE)
+            .field(Names.PATH, path)
+            .endObject();
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
+            throws MapperParsingException {
+            FieldAliasMapper.Builder builder = new FieldAliasMapper.Builder(name);
+            Object pathField = node.remove(Names.PATH);
+            String path = XContentMapValues.nodeStringValue(pathField, null);
+            if (path == null) {
+                throw new MapperParsingException("The [path] property must be specified for field [" + name + "].");
+            }
+            return builder.path(path);
+        }
+    }
+
+    public static class Builder extends Mapper.Builder<FieldAliasMapper.Builder, FieldAliasMapper> {
+        private String name;
+        private String path;
+
+        protected Builder(String name) {
+            super(name);
+            this.name = name;
+        }
+
+        public String name() {
+            return this.name;
+        }
+
+        public Builder path(String path) {
+            this.path = path;
+            return this;
+        }
+
+        public FieldAliasMapper build(BuilderContext context) {
+            String fullName = context.path().pathAsText(name);
+            return new FieldAliasMapper(name, fullName, path);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index 977b930c41e..cbb008c9d00 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -247,6 +247,11 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         return fieldType().name();
     }
 
+    @Override
+    public String typeName() {
+        return fieldType.typeName();
+    }
+
     public MappedFieldType fieldType() {
         return fieldType;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
index 069468ddb7a..c7d92e9f829 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
@@ -35,64 +35,115 @@ import java.util.Set;
  */
 class FieldTypeLookup implements Iterable<MappedFieldType> {
 
-    /** Full field name to field type */
     final CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType;
+    private final CopyOnWriteHashMap<String, String> aliasToConcreteName;
 
-    /** Create a new empty instance. */
     FieldTypeLookup() {
         fullNameToFieldType = new CopyOnWriteHashMap<>();
+        aliasToConcreteName = new CopyOnWriteHashMap<>();
     }
 
-    private FieldTypeLookup(CopyOnWriteHashMap<String, MappedFieldType> fullName) {
-        this.fullNameToFieldType = fullName;
+    private FieldTypeLookup(CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType,
+                            CopyOnWriteHashMap<String, String> aliasToConcreteName) {
+        this.fullNameToFieldType = fullNameToFieldType;
+        this.aliasToConcreteName = aliasToConcreteName;
     }
 
     /**
      * Return a new instance that contains the union of this instance and the field types
-     * from the provided fields. If a field already exists, the field type will be updated
-     * to use the new mappers field type.
+     * from the provided mappers. If a field already exists, its field type will be updated
+     * to use the new type from the given field mapper. Similarly if an alias already
+     * exists, it will be updated to reference the field type from the new mapper.
      */
-    public FieldTypeLookup copyAndAddAll(String type, Collection<FieldMapper> fieldMappers) {
+    public FieldTypeLookup copyAndAddAll(String type,
+                                         Collection<FieldMapper> fieldMappers,
+                                         Collection<FieldAliasMapper> fieldAliasMappers) {
         Objects.requireNonNull(type, "type must not be null");
         if (MapperService.DEFAULT_MAPPING.equals(type)) {
             throw new IllegalArgumentException("Default mappings should not be added to the lookup");
         }
 
         CopyOnWriteHashMap<String, MappedFieldType> fullName = this.fullNameToFieldType;
+        CopyOnWriteHashMap<String, String> aliases = this.aliasToConcreteName;
 
         for (FieldMapper fieldMapper : fieldMappers) {
             MappedFieldType fieldType = fieldMapper.fieldType();
             MappedFieldType fullNameFieldType = fullName.get(fieldType.name());
 
-            if (fullNameFieldType == null) {
-                // introduction of a new field
-                fullName = fullName.copyAndPut(fieldType.name(), fieldMapper.fieldType());
-            } else {
-                // modification of an existing field
-                checkCompatibility(fullNameFieldType, fieldType);
-                if (fieldType.equals(fullNameFieldType) == false) {
-                    fullName = fullName.copyAndPut(fieldType.name(), fieldMapper.fieldType());
-                }
+            if (!Objects.equals(fieldType, fullNameFieldType)) {
+                validateField(fullNameFieldType, fieldType, aliases);
+                fullName = fullName.copyAndPut(fieldType.name(), fieldType);
             }
         }
-        return new FieldTypeLookup(fullName);
+
+        for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) {
+            String aliasName = fieldAliasMapper.name();
+            String path = fieldAliasMapper.path();
+
+            validateAlias(aliasName, path, aliases, fullName);
+            aliases = aliases.copyAndPut(aliasName, path);
+        }
+
+        return new FieldTypeLookup(fullName, aliases);
     }
 
     /**
-     * Checks if the given field type is compatible with an existing field type.
-     * An IllegalArgumentException is thrown in case of incompatibility.
+     * Checks that the new field type is valid.
      */
-    private void checkCompatibility(MappedFieldType existingFieldType, MappedFieldType newFieldType) {
-        List<String> conflicts = new ArrayList<>();
-        existingFieldType.checkCompatibility(newFieldType, conflicts);
-        if (conflicts.isEmpty() == false) {
-            throw new IllegalArgumentException("Mapper for [" + newFieldType.name() + "] conflicts with existing mapping:\n" + conflicts.toString());
+    private void validateField(MappedFieldType existingFieldType,
+                               MappedFieldType newFieldType,
+                               CopyOnWriteHashMap<String, String> aliasToConcreteName) {
+        String fieldName = newFieldType.name();
+        if (aliasToConcreteName.containsKey(fieldName)) {
+            throw new IllegalArgumentException("The name for field [" + fieldName + "] has already" +
+                " been used to define a field alias.");
+        }
+
+        if (existingFieldType != null) {
+            List<String> conflicts = new ArrayList<>();
+            existingFieldType.checkCompatibility(newFieldType, conflicts);
+            if (conflicts.isEmpty() == false) {
+                throw new IllegalArgumentException("Mapper for [" + fieldName +
+                    "] conflicts with existing mapping:\n" + conflicts.toString());
+            }
+        }
+    }
+
+    /**
+     * Checks that the new field alias is valid.
+     *
+     * Note that this method assumes that new concrete fields have already been processed, so that it
+     * can verify that an alias refers to an existing concrete field.
+     */
+    private void validateAlias(String aliasName,
+                               String path,
+                               CopyOnWriteHashMap<String, String> aliasToConcreteName,
+                               CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType) {
+        if (fullNameToFieldType.containsKey(aliasName)) {
+            throw new IllegalArgumentException("The name for field alias [" + aliasName + "] has already" +
+                " been used to define a concrete field.");
+        }
+
+        if (path.equals(aliasName)) {
+            throw new IllegalArgumentException("Invalid [path] value [" + path + "] for field alias [" +
+                aliasName + "]: an alias cannot refer to itself.");
+        }
+
+        if (aliasToConcreteName.containsKey(path)) {
+            throw new IllegalArgumentException("Invalid [path] value [" + path + "] for field alias [" +
+                aliasName + "]: an alias cannot refer to another alias.");
+        }
+
+        if (!fullNameToFieldType.containsKey(path)) {
+            throw new IllegalArgumentException("Invalid [path] value [" + path + "] for field alias [" +
+                aliasName + "]: an alias must refer to an existing field in the mappings.");
         }
     }
 
     /** Returns the field for the given field */
     public MappedFieldType get(String field) {
-        return fullNameToFieldType.get(field);
+        String concreteField = aliasToConcreteName.getOrDefault(field, field);
+        return fullNameToFieldType.get(concreteField);
     }
 
     /**
@@ -105,6 +156,11 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
                 fields.add(fieldType.name());
             }
         }
+        for (String aliasName : aliasToConcreteName.keySet()) {
+            if (Regex.simpleMatch(pattern, aliasName)) {
+                fields.add(aliasName);
+            }
+        }
         return fields;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
index 051ac9da7f2..4d17afae614 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
@@ -173,6 +173,11 @@ public abstract class Mapper implements ToXContentFragment, Iterable<Mapper> {
     /** Returns the canonical name which uniquely identifies the mapper against other mappers in a type. */
     public abstract String name();
 
+    /**
+     * Returns a name representing the the type of this mapper.
+     */
+    public abstract String typeName();
+
     /** Return the merge of {@code mergeWith} into this.
      *  Both {@code this} and {@code mergeWith} will be left unmodified. */
     public abstract Mapper merge(Mapper mergeWith);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java
new file mode 100644
index 00000000000..440be98ad9e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Stream;
+
+/**
+ * A utility class that helps validate certain aspects of a mappings update.
+ */
+class MapperMergeValidator {
+
+    /**
+     * Validates the overall structure of the mapping addition, including whether
+     * duplicate fields are present, and if the provided fields have already been
+     * defined with a different data type.
+     *
+     * @param type The mapping type, for use in error messages.
+     * @param objectMappers The newly added object mappers.
+     * @param fieldMappers The newly added field mappers.
+     * @param fieldAliasMappers The newly added field alias mappers.
+     * @param fullPathObjectMappers All object mappers, indexed by their full path.
+     * @param fieldTypes All field and field alias mappers, collected into a lookup structure.
+     */
+    public static void validateMapperStructure(String type,
+                                               Collection<ObjectMapper> objectMappers,
+                                               Collection<FieldMapper> fieldMappers,
+                                               Collection<FieldAliasMapper> fieldAliasMappers,
+                                               Map<String, ObjectMapper> fullPathObjectMappers,
+                                               FieldTypeLookup fieldTypes) {
+        checkFieldUniqueness(type, objectMappers, fieldMappers,
+            fieldAliasMappers, fullPathObjectMappers, fieldTypes);
+        checkObjectsCompatibility(objectMappers, fullPathObjectMappers);
+    }
+
+    private static void checkFieldUniqueness(String type,
+                                             Collection<ObjectMapper> objectMappers,
+                                             Collection<FieldMapper> fieldMappers,
+                                             Collection<FieldAliasMapper> fieldAliasMappers,
+                                             Map<String, ObjectMapper> fullPathObjectMappers,
+                                             FieldTypeLookup fieldTypes) {
+
+        // first check within mapping
+        Set<String> objectFullNames = new HashSet<>();
+        for (ObjectMapper objectMapper : objectMappers) {
+            String fullPath = objectMapper.fullPath();
+            if (objectFullNames.add(fullPath) == false) {
+                throw new IllegalArgumentException("Object mapper [" + fullPath + "] is defined twice in mapping for type [" + type + "]");
+            }
+        }
+
+        Set<String> fieldNames = new HashSet<>();
+        Stream.concat(fieldMappers.stream(), fieldAliasMappers.stream())
+            .forEach(mapper -> {
+                String name = mapper.name();
+                if (objectFullNames.contains(name)) {
+                    throw new IllegalArgumentException("Field [" + name + "] is defined both as an object and a field in [" + type + "]");
+                } else if (fieldNames.add(name) == false) {
+                    throw new IllegalArgumentException("Field [" + name + "] is defined twice in [" + type + "]");
+                }
+            });
+
+        // then check other types
+        for (String fieldName : fieldNames) {
+            if (fullPathObjectMappers.containsKey(fieldName)) {
+                throw new IllegalArgumentException("[" + fieldName + "] is defined as a field in mapping [" + type
+                    + "] but this name is already used for an object in other types");
+            }
+        }
+
+        for (String objectPath : objectFullNames) {
+            if (fieldTypes.get(objectPath) != null) {
+                throw new IllegalArgumentException("[" + objectPath + "] is defined as an object in mapping [" + type
+                    + "] but this name is already used for a field in other types");
+            }
+        }
+    }
+
+    private static void checkObjectsCompatibility(Collection<ObjectMapper> objectMappers,
+                                                  Map<String, ObjectMapper> fullPathObjectMappers) {
+        for (ObjectMapper newObjectMapper : objectMappers) {
+            ObjectMapper existingObjectMapper = fullPathObjectMappers.get(newObjectMapper.fullPath());
+            if (existingObjectMapper != null) {
+                // simulate a merge and ignore the result, we are just interested
+                // in exceptions here
+                existingObjectMapper.merge(newObjectMapper);
+            }
+        }
+    }
+
+    /**
+     * Verifies that each field reference, e.g. the value of copy_to or the target
+     * of a field alias, corresponds to a valid part of the mapping.
+     *
+     * @param fieldMappers The newly added field mappers.
+     * @param fieldAliasMappers The newly added field alias mappers.
+     * @param fullPathObjectMappers All object mappers, indexed by their full path.
+     * @param fieldTypes All field and field alias mappers, collected into a lookup structure.
+     */
+    public static void validateFieldReferences(List<FieldMapper> fieldMappers,
+                                               List<FieldAliasMapper> fieldAliasMappers,
+                                               Map<String, ObjectMapper> fullPathObjectMappers,
+                                               FieldTypeLookup fieldTypes) {
+        validateCopyTo(fieldMappers, fullPathObjectMappers, fieldTypes);
+        validateFieldAliasTargets(fieldAliasMappers, fullPathObjectMappers);
+    }
+
+    private static void validateCopyTo(List<FieldMapper> fieldMappers,
+                                       Map<String, ObjectMapper> fullPathObjectMappers,
+                                       FieldTypeLookup fieldTypes) {
+        for (FieldMapper mapper : fieldMappers) {
+            if (mapper.copyTo() != null && mapper.copyTo().copyToFields().isEmpty() == false) {
+                String sourceParent = parentObject(mapper.name());
+                if (sourceParent != null && fieldTypes.get(sourceParent) != null) {
+                    throw new IllegalArgumentException("[copy_to] may not be used to copy from a multi-field: [" + mapper.name() + "]");
+                }
+
+                final String sourceScope = getNestedScope(mapper.name(), fullPathObjectMappers);
+                for (String copyTo : mapper.copyTo().copyToFields()) {
+                    String copyToParent = parentObject(copyTo);
+                    if (copyToParent != null && fieldTypes.get(copyToParent) != null) {
+                        throw new IllegalArgumentException("[copy_to] may not be used to copy to a multi-field: [" + copyTo + "]");
+                    }
+
+                    if (fullPathObjectMappers.containsKey(copyTo)) {
+                        throw new IllegalArgumentException("Cannot copy to field [" + copyTo + "] since it is mapped as an object");
+                    }
+
+                    final String targetScope = getNestedScope(copyTo, fullPathObjectMappers);
+                    checkNestedScopeCompatibility(sourceScope, targetScope);
+                }
+            }
+        }
+    }
+
+    private static void validateFieldAliasTargets(List<FieldAliasMapper> fieldAliasMappers,
+                                                  Map<String, ObjectMapper> fullPathObjectMappers) {
+        for (FieldAliasMapper mapper : fieldAliasMappers) {
+            String aliasName = mapper.name();
+            String path = mapper.path();
+
+            String aliasScope = getNestedScope(aliasName, fullPathObjectMappers);
+            String pathScope = getNestedScope(path, fullPathObjectMappers);
+
+            if (!Objects.equals(aliasScope, pathScope)) {
+                StringBuilder message = new StringBuilder("Invalid [path] value [" + path + "] for field alias [" +
+                    aliasName + "]: an alias must have the same nested scope as its target. ");
+                message.append(aliasScope == null
+                    ? "The alias is not nested"
+                    : "The alias's nested scope is [" + aliasScope + "]");
+                message.append(", but ");
+                message.append(pathScope == null
+                    ? "the target is not nested."
+                    : "the target's nested scope is [" + pathScope + "].");
+                throw new IllegalArgumentException(message.toString());
+            }
+        }
+    }
+
+    private static String getNestedScope(String path, Map<String, ObjectMapper> fullPathObjectMappers) {
+        for (String parentPath = parentObject(path); parentPath != null; parentPath = parentObject(parentPath)) {
+            ObjectMapper objectMapper = fullPathObjectMappers.get(parentPath);
+            if (objectMapper != null && objectMapper.nested().isNested()) {
+                return parentPath;
+            }
+        }
+        return null;
+    }
+
+    private static void checkNestedScopeCompatibility(String source, String target) {
+        boolean targetIsParentOfSource;
+        if (source == null || target == null) {
+            targetIsParentOfSource = target == null;
+        } else {
+            targetIsParentOfSource = source.equals(target) || source.startsWith(target + ".");
+        }
+        if (targetIsParentOfSource == false) {
+            throw new IllegalArgumentException(
+                "Illegal combination of [copy_to] and [nested] mappings: [copy_to] may only copy data to the current nested " +
+                    "document or any of its parents, however one [copy_to] directive is trying to copy data from nested object [" +
+                    source + "] to [" + target + "]");
+        }
+    }
+
+    private static String parentObject(String field) {
+        int lastDot = field.lastIndexOf('.');
+        if (lastDot == -1) {
+            return null;
+        }
+        return field.substring(0, lastDot);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 8988238d927..936e7334002 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.hppc.ObjectHashSet;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
@@ -395,15 +394,17 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
             // check basic sanity of the new mapping
             List<ObjectMapper> objectMappers = new ArrayList<>();
             List<FieldMapper> fieldMappers = new ArrayList<>();
+            List<FieldAliasMapper> fieldAliasMappers = new ArrayList<>();
             Collections.addAll(fieldMappers, newMapper.mapping().metadataMappers);
-            MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers);
-            checkFieldUniqueness(newMapper.type(), objectMappers, fieldMappers, fullPathObjectMappers, fieldTypes);
-            checkObjectsCompatibility(objectMappers, fullPathObjectMappers);
+            MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers, fieldAliasMappers);
+
+            MapperMergeValidator.validateMapperStructure(newMapper.type(), objectMappers, fieldMappers,
+                fieldAliasMappers, fullPathObjectMappers, fieldTypes);
             checkPartitionedIndexConstraints(newMapper);
 
             // update lookup data-structures
             // this will in particular make sure that the merged fields are compatible with other types
-            fieldTypes = fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers);
+            fieldTypes = fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers, fieldAliasMappers);
 
             for (ObjectMapper objectMapper : objectMappers) {
                 if (fullPathObjectMappers == this.fullPathObjectMappers) {
@@ -417,7 +418,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
                 }
             }
 
-            validateCopyTo(fieldMappers, fullPathObjectMappers, fieldTypes);
+            MapperMergeValidator.validateFieldReferences(fieldMappers, fieldAliasMappers,
+                fullPathObjectMappers, fieldTypes);
 
             if (reason == MergeReason.MAPPING_UPDATE) {
                 // this check will only be performed on the master node when there is
@@ -482,7 +484,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         if (mapper != null) {
             List<FieldMapper> fieldMappers = new ArrayList<>();
             Collections.addAll(fieldMappers, mapper.mapping().metadataMappers);
-            MapperUtils.collect(mapper.root(), new ArrayList<>(), fieldMappers);
+            MapperUtils.collect(mapper.root(), new ArrayList<>(), fieldMappers, new ArrayList<>());
             for (FieldMapper fieldMapper : fieldMappers) {
                 assert fieldMapper.fieldType() == fieldTypes.get(fieldMapper.name()) : fieldMapper.name();
             }
@@ -503,56 +505,6 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         return true;
     }
 
-    private static void checkFieldUniqueness(String type, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers,
-                                             Map<String, ObjectMapper> fullPathObjectMappers, FieldTypeLookup fieldTypes) {
-
-        // first check within mapping
-        final Set<String> objectFullNames = new HashSet<>();
-        for (ObjectMapper objectMapper : objectMappers) {
-            final String fullPath = objectMapper.fullPath();
-            if (objectFullNames.add(fullPath) == false) {
-                throw new IllegalArgumentException("Object mapper [" + fullPath + "] is defined twice in mapping for type [" + type + "]");
-            }
-        }
-
-        final Set<String> fieldNames = new HashSet<>();
-        for (FieldMapper fieldMapper : fieldMappers) {
-            final String name = fieldMapper.name();
-            if (objectFullNames.contains(name)) {
-                throw new IllegalArgumentException("Field [" + name + "] is defined both as an object and a field in [" + type + "]");
-            } else if (fieldNames.add(name) == false) {
-                throw new IllegalArgumentException("Field [" + name + "] is defined twice in [" + type + "]");
-            }
-        }
-
-        // then check other types
-        for (String fieldName : fieldNames) {
-            if (fullPathObjectMappers.containsKey(fieldName)) {
-                throw new IllegalArgumentException("[" + fieldName + "] is defined as a field in mapping [" + type
-                        + "] but this name is already used for an object in other types");
-            }
-        }
-
-        for (String objectPath : objectFullNames) {
-            if (fieldTypes.get(objectPath) != null) {
-                throw new IllegalArgumentException("[" + objectPath + "] is defined as an object in mapping [" + type
-                        + "] but this name is already used for a field in other types");
-            }
-        }
-    }
-
-    private static void checkObjectsCompatibility(Collection<ObjectMapper> objectMappers,
-                                                  Map<String, ObjectMapper> fullPathObjectMappers) {
-        for (ObjectMapper newObjectMapper : objectMappers) {
-            ObjectMapper existingObjectMapper = fullPathObjectMappers.get(newObjectMapper.fullPath());
-            if (existingObjectMapper != null) {
-                // simulate a merge and ignore the result, we are just interested
-                // in exceptions here
-                existingObjectMapper.merge(newObjectMapper);
-            }
-        }
-    }
-
     private void checkNestedFieldsLimit(Map<String, ObjectMapper> fullPathObjectMappers) {
         long allowedNestedFields = indexSettings.getValue(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
         long actualNestedFields = 0;
@@ -609,66 +561,6 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         }
     }
 
-    private static void validateCopyTo(List<FieldMapper> fieldMappers, Map<String, ObjectMapper> fullPathObjectMappers,
-            FieldTypeLookup fieldTypes) {
-        for (FieldMapper mapper : fieldMappers) {
-            if (mapper.copyTo() != null && mapper.copyTo().copyToFields().isEmpty() == false) {
-                String sourceParent = parentObject(mapper.name());
-                if (sourceParent != null && fieldTypes.get(sourceParent) != null) {
-                    throw new IllegalArgumentException("[copy_to] may not be used to copy from a multi-field: [" + mapper.name() + "]");
-                }
-
-                final String sourceScope = getNestedScope(mapper.name(), fullPathObjectMappers);
-                for (String copyTo : mapper.copyTo().copyToFields()) {
-                    String copyToParent = parentObject(copyTo);
-                    if (copyToParent != null && fieldTypes.get(copyToParent) != null) {
-                        throw new IllegalArgumentException("[copy_to] may not be used to copy to a multi-field: [" + copyTo + "]");
-                    }
-
-                    if (fullPathObjectMappers.containsKey(copyTo)) {
-                        throw new IllegalArgumentException("Cannot copy to field [" + copyTo + "] since it is mapped as an object");
-                    }
-
-                    final String targetScope = getNestedScope(copyTo, fullPathObjectMappers);
-                    checkNestedScopeCompatibility(sourceScope, targetScope);
-                }
-            }
-        }
-    }
-
-    private static String getNestedScope(String path, Map<String, ObjectMapper> fullPathObjectMappers) {
-        for (String parentPath = parentObject(path); parentPath != null; parentPath = parentObject(parentPath)) {
-            ObjectMapper objectMapper = fullPathObjectMappers.get(parentPath);
-            if (objectMapper != null && objectMapper.nested().isNested()) {
-                return parentPath;
-            }
-        }
-        return null;
-    }
-
-    private static void checkNestedScopeCompatibility(String source, String target) {
-        boolean targetIsParentOfSource;
-        if (source == null || target == null) {
-            targetIsParentOfSource = target == null;
-        } else {
-            targetIsParentOfSource = source.equals(target) || source.startsWith(target + ".");
-        }
-        if (targetIsParentOfSource == false) {
-            throw new IllegalArgumentException(
-                    "Illegal combination of [copy_to] and [nested] mappings: [copy_to] may only copy data to the current nested " +
-                            "document or any of its parents, however one [copy_to] directive is trying to copy data from nested object [" +
-                            source + "] to [" + target + "]");
-        }
-    }
-
-    private static String parentObject(String field) {
-        int lastDot = field.lastIndexOf('.');
-        if (lastDot == -1) {
-            return null;
-        }
-        return field.substring(0, lastDot);
-    }
-
     public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
         return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null);
     }
@@ -729,6 +621,13 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         return fieldTypes.simpleMatchToFullName(pattern);
     }
 
+    /**
+     * Returns all mapped field types.
+     */
+    public Iterable<MappedFieldType> fieldTypes() {
+        return fieldTypes;
+    }
+
     public ObjectMapper getObjectMapper(String name) {
         return fullPathObjectMappers.get(name);
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java
index ad57d72b345..70da6b73f31 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java
@@ -24,17 +24,28 @@ import java.util.Collection;
 enum MapperUtils {
     ;
 
-    /** Split mapper and its descendants into object and field mappers. */
-    public static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
+    /**
+     * Splits the provided mapper and its descendants into object, field, and field alias mappers.
+     */
+    public static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers,
+                               Collection<FieldMapper> fieldMappers,
+                               Collection<FieldAliasMapper> fieldAliasMappers) {
         if (mapper instanceof RootObjectMapper) {
             // root mapper isn't really an object mapper
         } else if (mapper instanceof ObjectMapper) {
             objectMappers.add((ObjectMapper)mapper);
         } else if (mapper instanceof FieldMapper) {
             fieldMappers.add((FieldMapper)mapper);
+        } else if (mapper instanceof FieldAliasMapper) {
+            fieldAliasMappers.add((FieldAliasMapper) mapper);
+        } else {
+            throw new IllegalStateException("Unrecognized mapper type [" +
+                mapper.getClass().getSimpleName() + "].");
         }
+
+
         for (Mapper child : mapper) {
-            collect(child, objectMappers, fieldMappers);
+            collect(child, objectMappers, fieldMappers, fieldAliasMappers);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index c3e3e41798d..99ad3936700 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -359,6 +359,11 @@ public class ObjectMapper extends Mapper implements Cloneable {
         return this.fullPath;
     }
 
+    @Override
+    public String typeName() {
+        return CONTENT_TYPE;
+    }
+
     public boolean isEnabled() {
         return this.enabled;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
index 280df7cfa6a..7a2373e5ad8 100644
--- a/server/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
@@ -149,7 +149,7 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder<ExistsQueryBuilder>
         }
 
         if (context.indexVersionCreated().before(Version.V_6_1_0)) {
-            return newLegacyExistsQuery(fields);
+            return newLegacyExistsQuery(context, fields);
         }
 
         if (fields.size() == 1) {
@@ -164,22 +164,28 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder<ExistsQueryBuilder>
         return new ConstantScoreQuery(boolFilterBuilder.build());
     }
 
-    private static Query newLegacyExistsQuery(Collection<String> fields) {
+    private static Query newLegacyExistsQuery(QueryShardContext context, Collection<String> fields) {
         // We create TermsQuery directly here rather than using FieldNamesFieldType.termsQuery()
         // so we don't end up with deprecation warnings
         if (fields.size() == 1) {
-            Query filter = new TermQuery(new Term(FieldNamesFieldMapper.NAME, fields.iterator().next()));
+            Query filter = newLegacyExistsQuery(context, fields.iterator().next());
             return new ConstantScoreQuery(filter);
         }
 
         BooleanQuery.Builder boolFilterBuilder = new BooleanQuery.Builder();
         for (String field : fields) {
-            Query filter = new TermQuery(new Term(FieldNamesFieldMapper.NAME, field));
+            Query filter = newLegacyExistsQuery(context, field);
             boolFilterBuilder.add(filter, BooleanClause.Occur.SHOULD);
         }
         return new ConstantScoreQuery(boolFilterBuilder.build());
     }
 
+    private static Query newLegacyExistsQuery(QueryShardContext context, String field) {
+        MappedFieldType fieldType = context.fieldMapper(field);
+        String fieldName = fieldType != null ? fieldType.name() : field;
+        return new TermQuery(new Term(FieldNamesFieldMapper.NAME, fieldName));
+    }
+
     private static Query newFieldExistsQuery(QueryShardContext context, String field) {
         MappedFieldType fieldType = context.getMapperService().fullName(field);
         if (fieldType == null) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
index 1f410a2564c..637d9321291 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
@@ -44,6 +44,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.io.IOException;
@@ -202,14 +203,18 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
                     multiTermQueryBuilder.getClass().getName() + ", should be " + MultiTermQuery.class.getName()
                     + " but was " + subQuery.getClass().getName());
             }
+
+            PrefixQueryBuilder prefixBuilder = (PrefixQueryBuilder) multiTermQueryBuilder;
+            MappedFieldType fieldType = context.fieldMapper(prefixBuilder.fieldName());
+            String fieldName = fieldType != null ? fieldType.name() : prefixBuilder.fieldName();
+
             if (context.getIndexSettings().getIndexVersionCreated().before(Version.V_6_4_0)) {
                 /**
                  * Indices created in this version do not index positions on the prefix field
                  * so we cannot use it to match positional queries. Instead, we explicitly create the prefix
                  * query on the main field to avoid the rewrite.
                  */
-                PrefixQueryBuilder prefixBuilder = (PrefixQueryBuilder) multiTermQueryBuilder;
-                PrefixQuery prefixQuery = new PrefixQuery(new Term(prefixBuilder.fieldName(), prefixBuilder.value()));
+                PrefixQuery prefixQuery = new PrefixQuery(new Term(fieldName, prefixBuilder.value()));
                 if (prefixBuilder.rewrite() != null) {
                     MultiTermQuery.RewriteMethod rewriteMethod =
                         QueryParsers.parseRewriteMethod(prefixBuilder.rewrite(), null, LoggingDeprecationHandler.INSTANCE);
@@ -218,15 +223,14 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
                 subQuery = prefixQuery;
                 spanQuery = new SpanMultiTermQueryWrapper<>(prefixQuery);
             } else {
-                String origFieldName = ((PrefixQueryBuilder) multiTermQueryBuilder).fieldName();
-                SpanTermQuery spanTermQuery = new SpanTermQuery(((TermQuery) subQuery).getTerm());
                 /**
                  * Prefixes are indexed in a different field so we mask the term query with the original field
                  * name. This is required because span_near and span_or queries don't work across different field.
                  * The masking is safe because the prefix field is indexed using the same content than the original field
                  * and the prefix analyzer preserves positions.
                  */
-                spanQuery = new FieldMaskingSpanQuery(spanTermQuery, origFieldName);
+                SpanTermQuery spanTermQuery = new SpanTermQuery(((TermQuery) subQuery).getTerm());
+                spanQuery = new FieldMaskingSpanQuery(spanTermQuery, fieldName);
             }
         } else {
             if (subQuery instanceof MultiTermQuery == false) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
index d4333fa0bc5..ceeef6112ae 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -218,7 +219,8 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
         }
         String spanNearFieldName = null;
         if (isGap) {
-            spanNearFieldName = ((SpanGapQueryBuilder) queryBuilder).fieldName();
+            String fieldName = ((SpanGapQueryBuilder) queryBuilder).fieldName();
+            spanNearFieldName = queryFieldName(context, fieldName);
         } else {
             spanNearFieldName = ((SpanQuery) query).getField();
         }
@@ -241,7 +243,9 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
             isGap = queryBuilder instanceof SpanGapQueryBuilder;
             if (isGap) {
                 String fieldName = ((SpanGapQueryBuilder) queryBuilder).fieldName();
-                if (!spanNearFieldName.equals(fieldName)) {
+                String spanGapFieldName = queryFieldName(context, fieldName);
+
+                if (!spanNearFieldName.equals(spanGapFieldName)) {
                     throw new IllegalArgumentException("[span_near] clauses must have same field");
                 }
                 int gap = ((SpanGapQueryBuilder) queryBuilder).width();
@@ -255,6 +259,11 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
         return builder.build();
     }
 
+    private String queryFieldName(QueryShardContext context, String fieldName) {
+        MappedFieldType fieldType = context.fieldMapper(fieldName);
+        return fieldType != null ? fieldType.name() : fieldName;
+    }
+
     @Override
     protected int doHashCode() {
         return Objects.hash(clauses, slop, inOrder);
@@ -273,11 +282,11 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
     }
 
     /**
-     * SpanGapQueryBuilder enables gaps in a SpanNearQuery. 
+     * SpanGapQueryBuilder enables gaps in a SpanNearQuery.
      * Since, SpanGapQuery is private to SpanNearQuery, SpanGapQueryBuilder cannot
      * be used to generate a Query (SpanGapQuery) like another QueryBuilder.
-     * Instead, it just identifies a span_gap clause so that SpanNearQuery.addGap(int) 
-     * can be invoked for it. 
+     * Instead, it just identifies a span_gap clause so that SpanNearQuery.addGap(int)
+     * can be invoked for it.
      * This QueryBuilder is only applicable as a clause in SpanGapQueryBuilder but
      * yet to enforce this restriction.
      */
@@ -286,9 +295,9 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
 
         /** Name of field to match against. */
         private final String fieldName;
-    
+
         /** Width of the gap introduced. */
-        private final int width; 
+        private final int width;
 
         /**
          * Constructs a new SpanGapQueryBuilder term query.
@@ -301,7 +310,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
                 throw new IllegalArgumentException("[span_gap] field name is null or empty");
             }
             //lucene has not coded any restriction on value of width.
-            //to-do : find if theoretically it makes sense to apply restrictions. 
+            //to-do : find if theoretically it makes sense to apply restrictions.
             this.fieldName = fieldName;
             this.width = width;
         }
@@ -396,7 +405,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
                     fieldName = currentFieldName;
                 } else if (token.isValue()) {
                     width = parser.intValue();
-                } 
+                }
             }
             SpanGapQueryBuilder result = new SpanGapQueryBuilder(fieldName, width);
             return result;
@@ -420,7 +429,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
             return Objects.hash(getClass(), fieldName, width);
         }
 
-        
+
         @Override
         public final String toString() {
             return Strings.toString(this, true, true);
diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
index fbada58f294..c20df00a109 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
@@ -221,7 +221,7 @@ public final class TermsSetQueryBuilder extends AbstractQueryBuilder<TermsSetQue
     }
 
     @Override
-    protected Query doToQuery(QueryShardContext context) throws IOException {
+    protected Query doToQuery(QueryShardContext context) {
         if (values.isEmpty()) {
             return Queries.newMatchNoDocsQuery("No terms supplied for \"" + getName() + "\" query.");
         }
@@ -230,6 +230,15 @@ public final class TermsSetQueryBuilder extends AbstractQueryBuilder<TermsSetQue
             throw new BooleanQuery.TooManyClauses();
         }
 
+        List<Query> queries = createTermQueries(context);
+        LongValuesSource longValuesSource = createValuesSource(context);
+        return new CoveringQuery(queries, longValuesSource);
+    }
+
+    /**
+     * Visible only for testing purposes.
+     */
+    List<Query> createTermQueries(QueryShardContext context) {
         final MappedFieldType fieldType = context.fieldMapper(fieldName);
         final List<Query> queries = new ArrayList<>(values.size());
         for (Object value : values) {
@@ -239,7 +248,11 @@ public final class TermsSetQueryBuilder extends AbstractQueryBuilder<TermsSetQue
                 queries.add(new TermQuery(new Term(fieldName, BytesRefs.toBytesRef(value))));
             }
         }
-        final LongValuesSource longValuesSource;
+        return queries;
+    }
+
+    private LongValuesSource createValuesSource(QueryShardContext context) {
+        LongValuesSource longValuesSource;
         if (minimumShouldMatchField != null) {
             MappedFieldType msmFieldType = context.fieldMapper(minimumShouldMatchField);
             if (msmFieldType == null) {
@@ -253,13 +266,13 @@ public final class TermsSetQueryBuilder extends AbstractQueryBuilder<TermsSetQue
                 SearchScript.TERMS_SET_QUERY_CONTEXT);
             Map<String, Object> params = new HashMap<>();
             params.putAll(minimumShouldMatchScript.getParams());
-            params.put("num_terms", queries.size());
+            params.put("num_terms", values.size());
             SearchScript.LeafFactory leafFactory = factory.newFactory(params, context.lookup());
             longValuesSource = new ScriptLongValueSource(minimumShouldMatchScript, leafFactory);
         } else {
             throw new IllegalStateException("No minimum should match has been specified");
         }
-        return new CoveringQuery(queries, longValuesSource);
+        return longValuesSource;
     }
 
     static final class ScriptLongValueSource extends LongValuesSource {
diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
index b3751afbc9c..df96ff87ec2 100644
--- a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
+++ b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
@@ -25,6 +25,8 @@ import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -88,10 +90,10 @@ public final class QueryParserHelper {
      * @param mapperService The mapper service where to find the mapping.
      * @param field The field name to search.
      */
-    public static FieldMapper getFieldMapper(MapperService mapperService, String field) {
+    public static Mapper getFieldMapper(MapperService mapperService, String field) {
         DocumentMapper mapper = mapperService.documentMapper();
         if (mapper != null) {
-            FieldMapper fieldMapper = mapper.mappers().getMapper(field);
+            Mapper fieldMapper = mapper.mappers().getMapper(field);
             if (fieldMapper != null) {
                 return fieldMapper;
             }
@@ -167,23 +169,27 @@ public final class QueryParserHelper {
             if (fieldSuffix != null && context.fieldMapper(fieldName + fieldSuffix) != null) {
                 fieldName = fieldName + fieldSuffix;
             }
-            FieldMapper mapper = getFieldMapper(context.getMapperService(), fieldName);
-            if (mapper == null) {
-                // Unmapped fields are not ignored
-                fields.put(fieldOrPattern, weight);
-                continue;
-            }
-            if (acceptMetadataField == false && mapper instanceof MetadataFieldMapper) {
-                // Ignore metadata fields
+
+            MappedFieldType fieldType = context.getMapperService().fullName(fieldName);
+            if (fieldType == null) {
+                // Note that we don't ignore unmapped fields.
+                fields.put(fieldName, weight);
                 continue;
             }
+
             // Ignore fields that are not in the allowed mapper types. Some
             // types do not support term queries, and thus we cannot generate
             // a special query for them.
-            String mappingType = mapper.fieldType().typeName();
+            String mappingType = fieldType.typeName();
             if (acceptAllTypes == false && ALLOWED_QUERY_MAPPER_TYPES.contains(mappingType) == false) {
                 continue;
             }
+
+            // Ignore metadata fields.
+            Mapper mapper = getFieldMapper(context.getMapperService(), fieldName);
+            if (acceptMetadataField == false && mapper instanceof MetadataFieldMapper) {
+                continue;
+            }
             fields.put(fieldName, weight);
         }
         checkForTooManyFields(fields);
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
index 2d41491e3a7..a1038853c06 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.FieldAliasMapper;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
@@ -129,7 +130,9 @@ public class IndicesModule extends AbstractModule {
         mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser());
         mappers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser());
         mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser());
+        mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser());
         mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
+
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
             mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index abdc195b514..df1bd115e2b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -85,6 +85,12 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
                                              AggregatorFactories.Builder subFactoriesBuilder,
                                              Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
+
+        if (!config.unmapped()) {
+            this.fieldType = config.fieldContext().fieldType();
+            this.indexedFieldName = fieldType.name();
+        }
+
         this.includeExclude = includeExclude;
         this.executionHint = executionHint;
         this.filter = filterBuilder == null
@@ -98,15 +104,6 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
                 : searcher.count(filter);
         this.bucketCountThresholds = bucketCountThresholds;
         this.significanceHeuristic = significanceHeuristic;
-        setFieldInfo(context);
-
-    }
-
-    private void setFieldInfo(SearchContext context) {
-        if (!config.unmapped()) {
-            this.indexedFieldName = config.fieldContext().field();
-            fieldType = context.smartNameFieldType(indexedFieldName);
-        }
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java
index 5e8bc2f4c18..f0b85f979c2 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java
@@ -343,13 +343,10 @@ public class SignificantTextAggregationBuilder extends AbstractAggregationBuilde
     protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent,
             Builder subFactoriesBuilder) throws IOException {
         SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(context);
-        String[] execFieldNames = sourceFieldNames;
-        if (execFieldNames == null) {
-            execFieldNames = new String[] { fieldName };
-        }
+
         return new SignificantTextAggregatorFactory(name, includeExclude, filterBuilder,
                 bucketCountThresholds, executionHeuristic, context, parent, subFactoriesBuilder,
-                fieldName, execFieldNames, filterDuplicateText, metaData);
+                fieldName, sourceFieldNames, filterDuplicateText, metaData);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
index c35b0bfd2d0..ea9a8a91aea 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
@@ -71,12 +71,19 @@ public class SignificantTextAggregatorFactory extends AggregatorFactory<Signific
             AggregatorFactories.Builder subFactoriesBuilder, String fieldName, String [] sourceFieldNames,
             boolean filterDuplicateText, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metaData);
+
+        // Note that if the field is unmapped (its field type is null), we don't fail,
+        // and just use the given field name as a placeholder.
+        this.fieldType = context.getQueryShardContext().fieldMapper(fieldName);
+        this.indexedFieldName = fieldType != null ? fieldType.name() : fieldName;
+        this.sourceFieldNames = sourceFieldNames == null
+            ? new String[] { indexedFieldName }
+            : sourceFieldNames;
+
         this.includeExclude = includeExclude;
         this.filter = filterBuilder == null
                 ? null
                 : filterBuilder.toQuery(context.getQueryShardContext());
-        this.indexedFieldName = fieldName;
-        this.sourceFieldNames = sourceFieldNames;
         this.filterDuplicateText = filterDuplicateText;
         IndexSearcher searcher = context.searcher();
         // Important - need to use the doc count that includes deleted docs
@@ -86,11 +93,8 @@ public class SignificantTextAggregatorFactory extends AggregatorFactory<Signific
                 : searcher.count(filter);
         this.bucketCountThresholds = bucketCountThresholds;
         this.significanceHeuristic = significanceHeuristic;
-        fieldType = context.getQueryShardContext().fieldMapper(indexedFieldName);
-
     }
 
-
     /**
      * Get the number of docs in the superset.
      */
@@ -133,13 +137,13 @@ public class SignificantTextAggregatorFactory extends AggregatorFactory<Signific
         }
         return context.searcher().count(query);
     }
-    
+
     public long getBackgroundFrequency(BytesRef termBytes) throws IOException {
         String value = format.format(termBytes).toString();
         return getBackgroundFrequency(value);
-    }    
+    }
+
 
-    
     @Override
     public void close() {
         try {
@@ -154,11 +158,11 @@ public class SignificantTextAggregatorFactory extends AggregatorFactory<Signific
     @Override
     protected Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
-            throws IOException {        
+            throws IOException {
         if (collectsFromSingleBucket == false) {
             return asMultiBucketAggregator(this, context, parent);
         }
-        
+
         numberOfAggregatorsCreated++;
         BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds);
         if (bucketCountThresholds.getShardSize() == SignificantTextAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
@@ -166,7 +170,7 @@ public class SignificantTextAggregatorFactory extends AggregatorFactory<Signific
             // Use default heuristic to avoid any wrong-ranking caused by
             // distributed counting but request double the usual amount.
             // We typically need more than the number of "top" terms requested
-            // by other aggregations as the significance algorithm is in less 
+            // by other aggregations as the significance algorithm is in less
             // of a position to down-select at shard-level - some of the things
             // we want to find have only one occurrence on each shard and as
             // such are impossible to differentiate from non-significant terms
@@ -177,9 +181,9 @@ public class SignificantTextAggregatorFactory extends AggregatorFactory<Signific
 
 //        TODO - need to check with mapping that this is indeed a text field....
 
-        IncludeExclude.StringFilter incExcFilter = includeExclude == null ? null: 
+        IncludeExclude.StringFilter incExcFilter = includeExclude == null ? null:
             includeExclude.convertToStringFilter(DocValueFormat.RAW);
-        
+
         return new SignificantTextAggregator(name, factories, context, parent, pipelineAggregators, bucketCountThresholds,
                 incExcFilter, significanceHeuristic, this, indexedFieldName, sourceFieldNames, filterDuplicateText, metaData);
 
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 08251c6a73b..64ed5f44795 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -31,7 +31,6 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -55,7 +54,7 @@ import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.tasks.TaskCancelledException;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -84,8 +83,7 @@ public class FetchPhase implements SearchPhase {
     @Override
     public void execute(SearchContext context) {
         final FieldsVisitor fieldsVisitor;
-        Set<String> fieldNames = null;
-        List<String> fieldNamePatterns = null;
+        Map<String, Set<String>> storedToRequestedFields = new HashMap<>();
         StoredFieldsContext storedFieldsContext = context.storedFieldsContext();
 
         if (storedFieldsContext == null) {
@@ -98,39 +96,36 @@ public class FetchPhase implements SearchPhase {
             // disable stored fields entirely
             fieldsVisitor = null;
         } else {
-            for (String fieldName : context.storedFieldsContext().fieldNames()) {
-                if (fieldName.equals(SourceFieldMapper.NAME)) {
+            for (String fieldNameOrPattern : context.storedFieldsContext().fieldNames()) {
+                if (fieldNameOrPattern.equals(SourceFieldMapper.NAME)) {
                     FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() ? context.fetchSourceContext()
-                            : FetchSourceContext.FETCH_SOURCE;
+                        : FetchSourceContext.FETCH_SOURCE;
                     context.fetchSourceContext(new FetchSourceContext(true, fetchSourceContext.includes(), fetchSourceContext.excludes()));
                     continue;
                 }
-                if (Regex.isSimpleMatchPattern(fieldName)) {
-                    if (fieldNamePatterns == null) {
-                        fieldNamePatterns = new ArrayList<>();
-                    }
-                    fieldNamePatterns.add(fieldName);
-                } else {
+
+                Collection<String> fieldNames = context.mapperService().simpleMatchToFullName(fieldNameOrPattern);
+                for (String fieldName : fieldNames) {
                     MappedFieldType fieldType = context.smartNameFieldType(fieldName);
                     if (fieldType == null) {
                         // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
                         if (context.getObjectMapper(fieldName) != null) {
                             throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field");
                         }
+                    } else {
+                        String storedField = fieldType.name();
+                        Set<String> requestedFields = storedToRequestedFields.computeIfAbsent(
+                            storedField, key -> new HashSet<>());
+                        requestedFields.add(fieldName);
                     }
-                    if (fieldNames == null) {
-                        fieldNames = new HashSet<>();
-                    }
-                    fieldNames.add(fieldName);
                 }
             }
             boolean loadSource = context.sourceRequested();
-            if (fieldNames == null && fieldNamePatterns == null) {
+            if (storedToRequestedFields.isEmpty()) {
                 // empty list specified, default to disable _source if no explicit indication
                 fieldsVisitor = new FieldsVisitor(loadSource);
             } else {
-                fieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames,
-                        fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, loadSource);
+                fieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), loadSource);
             }
         }
 
@@ -149,10 +144,11 @@ public class FetchPhase implements SearchPhase {
                 final SearchHit searchHit;
                 int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId);
                 if (rootDocId != -1) {
-                    searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns,
-                            subReaderContext);
+                    searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId,
+                        storedToRequestedFields, subReaderContext);
                 } else {
-                    searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext);
+                    searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId,
+                        storedToRequestedFields, subReaderContext);
                 }
 
                 hits[index] = searchHit;
@@ -190,21 +186,18 @@ public class FetchPhase implements SearchPhase {
         return -1;
     }
 
-    private SearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVisitor, int docId, int subDocId,
+    private SearchHit createSearchHit(SearchContext context,
+                                      FieldsVisitor fieldsVisitor,
+                                      int docId,
+                                      int subDocId,
+                                      Map<String, Set<String>> storedToRequestedFields,
                                       LeafReaderContext subReaderContext) {
         if (fieldsVisitor == null) {
             return new SearchHit(docId);
         }
-        loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
-        fieldsVisitor.postProcess(context.mapperService());
 
-        Map<String, DocumentField> searchFields = null;
-        if (!fieldsVisitor.fields().isEmpty()) {
-            searchFields = new HashMap<>(fieldsVisitor.fields().size());
-            for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
-                searchFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
-            }
-        }
+        Map<String, DocumentField> searchFields = getSearchFields(context, fieldsVisitor, subDocId,
+            storedToRequestedFields, subReaderContext);
 
         DocumentMapper documentMapper = context.mapperService().documentMapper(fieldsVisitor.uid().type());
         Text typeText;
@@ -223,9 +216,40 @@ public class FetchPhase implements SearchPhase {
         return searchHit;
     }
 
-    private SearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId,
-                                            int rootSubDocId, Set<String> fieldNames,
-                                            List<String> fieldNamePatterns, LeafReaderContext subReaderContext) throws IOException {
+    private Map<String, DocumentField> getSearchFields(SearchContext context,
+                                                       FieldsVisitor fieldsVisitor,
+                                                       int subDocId,
+                                                       Map<String, Set<String>> storedToRequestedFields,
+                                                       LeafReaderContext subReaderContext) {
+        loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
+        fieldsVisitor.postProcess(context.mapperService());
+
+        if (fieldsVisitor.fields().isEmpty()) {
+            return null;
+        }
+
+        Map<String, DocumentField> searchFields = new HashMap<>(fieldsVisitor.fields().size());
+        for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
+            String storedField = entry.getKey();
+            List<Object> storedValues = entry.getValue();
+
+            if (storedToRequestedFields.containsKey(storedField)) {
+                for (String requestedField : storedToRequestedFields.get(storedField)) {
+                    searchFields.put(requestedField, new DocumentField(requestedField, storedValues));
+                }
+            } else {
+                searchFields.put(storedField, new DocumentField(storedField, storedValues));
+            }
+        }
+        return searchFields;
+    }
+
+    private SearchHit createNestedSearchHit(SearchContext context,
+                                            int nestedTopDocId,
+                                            int nestedSubDocId,
+                                            int rootSubDocId,
+                                            Map<String, Set<String>> storedToRequestedFields,
+                                            LeafReaderContext subReaderContext) throws IOException {
         // Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
         // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
         // because the entire _source is only stored with the root document.
@@ -244,9 +268,13 @@ public class FetchPhase implements SearchPhase {
             source = null;
         }
 
+        Map<String, DocumentField> searchFields = null;
+        if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
+            FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), false);
+            searchFields = getSearchFields(context, nestedFieldsVisitor, nestedSubDocId,
+                storedToRequestedFields, subReaderContext);
+        }
 
-        Map<String, DocumentField> searchFields =
-                getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext);
         DocumentMapper documentMapper = context.mapperService().documentMapper(uid.type());
         SourceLookup sourceLookup = context.lookup().source();
         sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);
@@ -307,26 +335,6 @@ public class FetchPhase implements SearchPhase {
         return new SearchHit(nestedTopDocId, uid.id(), documentMapper.typeText(), nestedIdentity, searchFields);
     }
 
-    private Map<String, DocumentField> getSearchFields(SearchContext context, int nestedSubDocId, Set<String> fieldNames,
-                                                       List<String> fieldNamePatterns, LeafReaderContext subReaderContext) {
-        Map<String, DocumentField> searchFields = null;
-        if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
-            FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames,
-                    fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, false);
-            if (nestedFieldsVisitor != null) {
-                loadStoredFields(context, subReaderContext, nestedFieldsVisitor, nestedSubDocId);
-                nestedFieldsVisitor.postProcess(context.mapperService());
-                if (!nestedFieldsVisitor.fields().isEmpty()) {
-                    searchFields = new HashMap<>(nestedFieldsVisitor.fields().size());
-                    for (Map.Entry<String, List<Object>> entry : nestedFieldsVisitor.fields().entrySet()) {
-                        searchFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
-                    }
-                }
-            }
-        }
-        return searchFields;
-    }
-
     private SearchHit.NestedIdentity getInternalNestedIdentity(SearchContext context, int nestedSubDocId,
                                                                LeafReaderContext subReaderContext,
                                                                MapperService mapperService,
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
index e5ff7abc68b..11e46061d67 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
@@ -100,7 +100,7 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
                 if (highlightQuery == null) {
                     highlightQuery = context.parsedQuery().query();
                 }
-                HighlighterContext highlighterContext = new HighlighterContext(fieldName,
+                HighlighterContext highlighterContext = new HighlighterContext(fieldType.name(),
                     field, fieldType, context, hitContext, highlightQuery);
 
                 if ((highlighter.canHighlight(fieldType) == false) && fieldNameContainsWildcards) {
@@ -109,7 +109,11 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
                 }
                 HighlightField highlightField = highlighter.highlight(highlighterContext);
                 if (highlightField != null) {
-                    highlightFields.put(highlightField.name(), highlightField);
+                    // Note that we make sure to use the original field name in the response. This is because the
+                    // original field could be an alias, and highlighter implementations may instead reference the
+                    // concrete field it points to.
+                    highlightFields.put(fieldName,
+                        new HighlightField(fieldName, highlightField.fragments()));
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
index 2e04443f9e5..9ec20001adc 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
@@ -148,7 +148,7 @@ public class LeafFieldsLookup implements Map {
                 reader.document(docId, fieldVisitor);
                 fieldVisitor.postProcess(mapperService);
                 List<Object> storedFields = fieldVisitor.fields().get(data.fieldType().name());
-                data.fields(singletonMap(name, storedFields));
+                data.fields(singletonMap(fieldName, storedFields));
             } catch (IOException e) {
                 throw new ElasticsearchParseException("failed to load field [{}]", e, name);
             }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
index dcdc669539f..9199615868a 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.BytesRefs;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -321,7 +320,7 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> implemen
             suggestionContext.setAnalyzer(luceneAnalyzer);
         }
 
-        suggestionContext.setField(field);
+        suggestionContext.setField(fieldType.name());
 
         if (size != null) {
             suggestionContext.setSize(size);
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
index c4f7d8a5000..48aaf705099 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java
@@ -29,8 +29,8 @@ import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 
@@ -138,8 +138,8 @@ public class GeoContextMapping extends ContextMapping<GeoQueryContext> {
     @Override
     public Set<CharSequence> parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException {
         if (fieldName != null) {
-            FieldMapper mapper = parseContext.docMapper().mappers().getMapper(fieldName);
-            if (!(mapper instanceof GeoPointFieldMapper)) {
+            MappedFieldType fieldType = parseContext.mapperService().fullName(fieldName);
+            if (!(fieldType instanceof GeoPointFieldMapper.GeoPointFieldType)) {
                 throw new ElasticsearchParseException("referenced field must be mapped to geo_point");
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java b/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java
index 8c4879fd35e..9aba48f7de5 100644
--- a/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java
@@ -21,12 +21,11 @@ package org.elasticsearch.index.analysis;
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.analysis.PreBuiltAnalyzers;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -84,14 +83,14 @@ public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase {
 
         NamedAnalyzer namedAnalyzer = new PreBuiltAnalyzerProvider(analyzerName, AnalyzerScope.INDEX, randomPreBuiltAnalyzer.getAnalyzer(randomVersion)).get();
 
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", "text").field("analyzer", analyzerName).endObject().endObject()
-                .endObject().endObject());
-        DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+                .endObject().endObject();
+        MapperService mapperService = createIndex("test", indexSettings, "type", mapping).mapperService();
 
-        FieldMapper fieldMapper = docMapper.mappers().getMapper("field");
-        assertThat(fieldMapper.fieldType().searchAnalyzer(), instanceOf(NamedAnalyzer.class));
-        NamedAnalyzer fieldMapperNamedAnalyzer = fieldMapper.fieldType().searchAnalyzer();
+        MappedFieldType fieldType = mapperService.fullName("field");
+        assertThat(fieldType.searchAnalyzer(), instanceOf(NamedAnalyzer.class));
+        NamedAnalyzer fieldMapperNamedAnalyzer = fieldType.searchAnalyzer();
 
         assertThat(fieldMapperNamedAnalyzer.analyzer(), is(namedAnalyzer.analyzer()));
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
index 6e9cb6c0b59..5f4cd98600b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
@@ -27,6 +27,8 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
@@ -49,32 +51,32 @@ public class BinaryFieldMapperTests extends ESSingleNodeTestCase {
     }
 
     public void testDefaultMapping() throws Exception {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties")
                 .startObject("field")
                 .field("type", "binary")
                 .endObject()
                 .endObject()
-                .endObject().endObject());
+                .endObject().endObject();
 
-        DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type", mapping).mapperService();
+        MappedFieldType fieldType = mapperService.fullName("field");
 
-        FieldMapper fieldMapper = mapper.mappers().getMapper("field");
-        assertThat(fieldMapper, instanceOf(BinaryFieldMapper.class));
-        assertThat(fieldMapper.fieldType().stored(), equalTo(false));
+        assertThat(fieldType, instanceOf(BinaryFieldMapper.BinaryFieldType.class));
+        assertThat(fieldType.stored(), equalTo(false));
     }
 
     public void testStoredValue() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties")
                 .startObject("field")
                 .field("type", "binary")
                 .field("store", true)
                 .endObject()
                 .endObject()
-                .endObject().endObject());
+                .endObject().endObject();
 
-        DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type", mapping).mapperService();
 
         // case 1: a simple binary value
         final byte[] binaryValue1 = new byte[100];
@@ -89,13 +91,14 @@ public class BinaryFieldMapperTests extends ESSingleNodeTestCase {
         assertTrue(CompressorFactory.isCompressed(new BytesArray(binaryValue2)));
 
         for (byte[] value : Arrays.asList(binaryValue1, binaryValue2)) {
-            ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "id",
+            ParsedDocument doc = mapperService.documentMapper().parse(SourceToParse.source("test", "type", "id",
                     BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", value).endObject()),
                     XContentType.JSON));
             BytesRef indexedValue = doc.rootDoc().getBinaryValue("field");
             assertEquals(new BytesRef(value), indexedValue);
-            FieldMapper fieldMapper = mapper.mappers().getMapper("field");
-            Object originalValue = fieldMapper.fieldType().valueForDisplay(indexedValue);
+
+            MappedFieldType fieldType = mapperService.fullName("field");
+            Object originalValue = fieldType.valueForDisplay(indexedValue);
             assertEquals(new BytesArray(value), originalValue);
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
index 44ecb24b672..8638e16e29a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
@@ -52,7 +52,6 @@ import static org.hamcrest.Matchers.containsString;
 public class BooleanFieldMapperTests extends ESSingleNodeTestCase {
     private IndexService indexService;
     private DocumentMapperParser parser;
-    private DocumentMapperParser preEs6Parser;
 
     @Before
     public void setup() {
@@ -101,7 +100,7 @@ public class BooleanFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
-        FieldMapper mapper = defaultMapper.mappers().getMapper("field");
+        Mapper mapper = defaultMapper.mappers().getMapper("field");
         XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
         mapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
index 1381b6e9205..a01ddccc939 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
@@ -32,6 +32,7 @@ import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -61,10 +62,9 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
 
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
-
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        MappedFieldType completionFieldType = ((CompletionFieldMapper) fieldMapper).fieldType();
 
         NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer();
         assertThat(indexAnalyzer.name(), equalTo("simple"));
@@ -94,10 +94,9 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
 
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
-
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        MappedFieldType completionFieldType = ((CompletionFieldMapper) fieldMapper).fieldType();
 
         NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer();
         assertThat(indexAnalyzer.name(), equalTo("simple"));
@@ -129,12 +128,11 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
 
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
 
-        CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
         XContentBuilder builder = jsonBuilder().startObject();
-        completionFieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject();
+        fieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject();
         builder.close();
         Map<String, Object> serializedMap = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder)).map();
         Map<String, Object> configMap = (Map<String, Object>) serializedMap.get("completion");
@@ -153,15 +151,15 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("completion", "suggestion")
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertSuggestFields(fields, 1);
     }
 
@@ -192,15 +190,15 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .array("completion", "suggestion1", "suggestion2")
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertSuggestFields(fields, 2);
     }
 
@@ -212,8 +210,8 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
                         .startObject()
@@ -223,7 +221,7 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                         .endObject()
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertSuggestFields(fields, 1);
     }
 
@@ -235,8 +233,8 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
                         .startObject()
@@ -246,10 +244,50 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                         .endObject()
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertSuggestFields(fields, 3);
     }
 
+    public void testParsingWithGeoFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("type1")
+                .startObject("properties")
+                    .startObject("completion")
+                        .field("type", "completion")
+                        .startObject("contexts")
+                            .field("name", "location")
+                            .field("type", "geo")
+                            .field("path", "alias")
+                        .endObject()
+                    .endObject()
+                    .startObject("birth-place")
+                        .field("type", "geo_point")
+                    .endObject()
+                    .startObject("alias")
+                        .field("type", "alias")
+                        .field("path", "birth-place")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject();
+
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type1", mapping).mapperService();
+        Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper("completion");
+
+        ParsedDocument parsedDocument = mapperService.documentMapper().parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("completion")
+                        .field("input", "suggestion")
+                        .startObject("contexts")
+                            .field("location", "37.77,-122.42")
+                        .endObject()
+                    .endObject()
+                .endObject()), XContentType.JSON));
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
+        assertSuggestFields(fields, 1);
+    }
+
     public void testParsingFull() throws Exception {
         String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("completion")
@@ -258,8 +296,8 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
                         .startObject()
@@ -279,7 +317,7 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                         .endArray()
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertSuggestFields(fields, 3);
     }
 
@@ -291,8 +329,8 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
                         .startObject()
@@ -312,7 +350,7 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                         .endArray()
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertSuggestFields(fields, 6);
     }
 
@@ -420,7 +458,7 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
         Query prefixQuery = completionFieldMapper.fieldType().prefixQuery(new BytesRef("co"));
         assertThat(prefixQuery, instanceOf(PrefixCompletionQuery.class));
@@ -434,7 +472,7 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
         Query prefixQuery = completionFieldMapper.fieldType().fuzzyQuery("co",
                 Fuzziness.fromEdits(FuzzyCompletionQuery.DEFAULT_MAX_EDITS), FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX,
@@ -451,7 +489,7 @@ public class CompletionFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
         Query prefixQuery = completionFieldMapper.fieldType()
                 .regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
index eff6222e6c6..5eb102208eb 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
@@ -72,7 +72,7 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
         IndexService index = createIndex("test");
         client().admin().indices().preparePutMapping("test").setType("type1").setSource(mapping, XContentType.JSON).get();
         DocumentMapper docMapper = index.mapperService().documentMapper("type1");
-        FieldMapper fieldMapper = docMapper.mappers().getMapper("copy_test");
+        Mapper fieldMapper = docMapper.mappers().getMapper("copy_test");
 
         // Check json serialization
         TextFieldMapper stringFieldMapper = (TextFieldMapper) fieldMapper;
@@ -123,7 +123,7 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
 
         docMapper = index.mapperService().documentMapper("type1");
         fieldMapper = docMapper.mappers().getMapper("new_field");
-        assertThat(fieldMapper.fieldType().typeName(), equalTo("long"));
+        assertThat(fieldMapper.typeName(), equalTo("long"));
     }
 
     public void testCopyToFieldsInnerObjectParsing() throws Exception {
@@ -308,13 +308,15 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
 
         MapperService mapperService = createIndex("test").mapperService();
         DocumentMapper docMapperBefore = mapperService.merge("type1", new CompressedXContent(mappingBefore), MapperService.MergeReason.MAPPING_UPDATE);
+        FieldMapper fieldMapperBefore = (FieldMapper) docMapperBefore.mappers().getMapper("copy_test");
 
-        assertEquals(Arrays.asList("foo", "bar"), docMapperBefore.mappers().getMapper("copy_test").copyTo().copyToFields());
+        assertEquals(Arrays.asList("foo", "bar"), fieldMapperBefore.copyTo().copyToFields());
 
         DocumentMapper docMapperAfter = mapperService.merge("type1", new CompressedXContent(mappingAfter), MapperService.MergeReason.MAPPING_UPDATE);
+        FieldMapper fieldMapperAfter = (FieldMapper) docMapperAfter.mappers().getMapper("copy_test");
 
-        assertEquals(Arrays.asList("baz", "bar"), docMapperAfter.mappers().getMapper("copy_test").copyTo().copyToFields());
-        assertEquals(Arrays.asList("foo", "bar"), docMapperBefore.mappers().getMapper("copy_test").copyTo().copyToFields());
+        assertEquals(Arrays.asList("baz", "bar"), fieldMapperAfter.copyTo().copyToFields());
+        assertEquals(Arrays.asList("foo", "bar"), fieldMapperBefore.copyTo().copyToFields());
     }
 
     public void testCopyToNestedField() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index c19965ac5f7..51b27094099 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -382,11 +382,11 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
             .startObject("properties")
             .startObject("release_date").field("type", "date").field("format", "yyyy/MM/dd").endObject()
             .endObject().endObject().endObject());
-        DocumentMapper initMapper = indexService.mapperService().merge("movie", new CompressedXContent(initMapping),
+        indexService.mapperService().merge("movie", new CompressedXContent(initMapping),
             MapperService.MergeReason.MAPPING_UPDATE);
 
-        assertThat(initMapper.mappers().getMapper("release_date"), notNullValue());
-        assertFalse(initMapper.mappers().getMapper("release_date").fieldType().stored());
+        assertThat(indexService.mapperService().fullName("release_date"), notNullValue());
+        assertFalse(indexService.mapperService().fullName("release_date").stored());
 
         String updateFormatMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("movie")
             .startObject("properties")
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
index 4e79a68c50e..4373f2210a7 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
@@ -39,6 +39,7 @@ import org.elasticsearch.index.query.QueryShardContext;
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 public class DocumentFieldMapperTests extends LuceneTestCase {
@@ -138,7 +139,12 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
         Analyzer defaultSearch = new FakeAnalyzer("default_search");
         Analyzer defaultSearchQuote = new FakeAnalyzer("default_search_quote");
 
-        DocumentFieldMappers documentFieldMappers = new DocumentFieldMappers(Arrays.asList(fieldMapper1, fieldMapper2), defaultIndex, defaultSearch, defaultSearchQuote);
+        DocumentFieldMappers documentFieldMappers = new DocumentFieldMappers(
+            Arrays.asList(fieldMapper1, fieldMapper2),
+            Collections.emptyList(),
+            defaultIndex,
+            defaultSearch,
+            defaultSearchQuote);
 
         assertAnalyzes(documentFieldMappers.indexAnalyzer(), "field1", "index");
         assertAnalyzes(documentFieldMappers.searchAnalyzer(), "field1", "search");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
index 0234fcb681d..54b6b2310da 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
@@ -23,9 +23,10 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
@@ -104,38 +105,50 @@ public class DocumentMapperMergeTests extends ESSingleNodeTestCase {
     }
 
     public void testMergeSearchAnalyzer() throws Exception {
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
-        String mapping1 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "text").field("analyzer", "standard").field("search_analyzer", "whitespace").endObject().endObject()
-                .endObject().endObject());
+        XContentBuilder mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field")
+                .field("type", "text")
+                .field("analyzer", "standard")
+                .field("search_analyzer", "whitespace")
+            .endObject().endObject()
+        .endObject().endObject();
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type", mapping1).mapperService();
+
+        assertThat(mapperService.fullName("field").searchAnalyzer().name(), equalTo("whitespace"));
+
         String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "text").field("analyzer", "standard").field("search_analyzer", "keyword").endObject().endObject()
-                .endObject().endObject());
+            .startObject("properties").startObject("field")
+                .field("type", "text")
+                .field("analyzer", "standard")
+                .field("search_analyzer", "keyword")
+            .endObject().endObject()
+        .endObject().endObject());
 
-        DocumentMapper existing = parser.parse("type", new CompressedXContent(mapping1));
-        DocumentMapper changed = parser.parse("type", new CompressedXContent(mapping2));
-
-        assertThat(((NamedAnalyzer) existing.mappers().getMapper("field").fieldType().searchAnalyzer()).name(), equalTo("whitespace"));
-        DocumentMapper merged = existing.merge(changed.mapping());
-
-        assertThat(((NamedAnalyzer) merged.mappers().getMapper("field").fieldType().searchAnalyzer()).name(), equalTo("keyword"));
+        mapperService.merge("type", new CompressedXContent(mapping2), MapperService.MergeReason.MAPPING_UPDATE);
+        assertThat(mapperService.fullName("field").searchAnalyzer().name(), equalTo("keyword"));
     }
 
     public void testChangeSearchAnalyzerToDefault() throws Exception {
-        MapperService mapperService = createIndex("test").mapperService();
-        String mapping1 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "text").field("analyzer", "standard").field("search_analyzer", "whitespace").endObject().endObject()
-                .endObject().endObject());
+          XContentBuilder mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field")
+                .field("type", "text")
+                .field("analyzer", "standard")
+                .field("search_analyzer", "whitespace")
+            .endObject().endObject()
+        .endObject().endObject();
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type", mapping1).mapperService();
+
+        assertThat(mapperService.fullName("field").searchAnalyzer().name(), equalTo("whitespace"));
+
         String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "text").field("analyzer", "standard").endObject().endObject()
-                .endObject().endObject());
+            .startObject("properties").startObject("field")
+                .field("type", "text")
+                .field("analyzer", "standard")
+            .endObject().endObject()
+        .endObject().endObject());
 
-        DocumentMapper existing = mapperService.merge("type", new CompressedXContent(mapping1), MapperService.MergeReason.MAPPING_UPDATE);
-        DocumentMapper merged = mapperService.merge("type", new CompressedXContent(mapping2), MapperService.MergeReason.MAPPING_UPDATE);
-
-        assertThat(((NamedAnalyzer) existing.mappers().getMapper("field").fieldType().searchAnalyzer()).name(), equalTo("whitespace"));
-
-        assertThat(((NamedAnalyzer) merged.mappers().getMapper("field").fieldType().searchAnalyzer()).name(), equalTo("standard"));
+        mapperService.merge("type", new CompressedXContent(mapping2), MapperService.MergeReason.MAPPING_UPDATE);
+        assertThat(mapperService.fullName("field").searchAnalyzer().name(), equalTo("standard"));
     }
 
     public void testConcurrentMergeTest() throws Throwable {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 11f69c738e9..098aacc799f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -1001,7 +1001,7 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
         BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json"));
         Document doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
 
-        assertThat(doc.get(docMapper.mappers().getMapper("name.first").fieldType().name()), equalTo("shay"));
+        assertThat(doc.get(docMapper.mappers().getMapper("name.first").name()), equalTo("shay"));
         doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
     }
 
@@ -1014,8 +1014,8 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
         DocumentMapper builtDocMapper = parser.parse("person", new CompressedXContent(builtMapping));
         BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json"));
         Document doc = builtDocMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
-        assertThat(doc.getBinaryValue(docMapper.idFieldMapper().fieldType().name()), equalTo(Uid.encodeId("1")));
-        assertThat(doc.get(docMapper.mappers().getMapper("name.first").fieldType().name()), equalTo("shay"));
+        assertThat(doc.getBinaryValue(docMapper.idFieldMapper().name()), equalTo(Uid.encodeId("1")));
+        assertThat(doc.get(docMapper.mappers().getMapper("name.first").name()), equalTo("shay"));
     }
 
     public void testSimpleParser() throws Exception {
@@ -1026,8 +1026,8 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
 
         BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json"));
         Document doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
-        assertThat(doc.getBinaryValue(docMapper.idFieldMapper().fieldType().name()), equalTo(Uid.encodeId("1")));
-        assertThat(doc.get(docMapper.mappers().getMapper("name.first").fieldType().name()), equalTo("shay"));
+        assertThat(doc.getBinaryValue(docMapper.idFieldMapper().name()), equalTo(Uid.encodeId("1")));
+        assertThat(doc.get(docMapper.mappers().getMapper("name.first").name()), equalTo("shay"));
     }
 
     public void testSimpleParserNoTypeNoId() throws Exception {
@@ -1035,8 +1035,8 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
         BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1-notype-noid.json"));
         Document doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
-        assertThat(doc.getBinaryValue(docMapper.idFieldMapper().fieldType().name()), equalTo(Uid.encodeId("1")));
-        assertThat(doc.get(docMapper.mappers().getMapper("name.first").fieldType().name()), equalTo("shay"));
+        assertThat(doc.getBinaryValue(docMapper.idFieldMapper().name()), equalTo(Uid.encodeId("1")));
+        assertThat(doc.get(docMapper.mappers().getMapper("name.first").name()), equalTo("shay"));
     }
 
     public void testAttributes() throws Exception {
@@ -1389,4 +1389,98 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
                 client().prepareIndex("idx", "type").setSource(bytes2, XContentType.JSON).get());
         assertThat(ExceptionsHelper.detailedMessage(err), containsString("field name cannot be an empty string"));
     }
+
+    public void testWriteToFieldAlias() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                            .field("path", "concrete-field")
+                        .endObject()
+                        .startObject("concrete-field")
+                            .field("type", "keyword")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder()
+            .startObject()
+                .field("alias-field", "value")
+            .endObject());
+        MapperParsingException exception = expectThrows(MapperParsingException.class,
+            () -> mapper.parse(SourceToParse.source("test", "type", "1", bytes, XContentType.JSON)));
+
+        assertEquals("Cannot write to a field alias [alias-field].", exception.getCause().getMessage());
+    }
+
+     public void testCopyToFieldAlias() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                            .field("path", "concrete-field")
+                        .endObject()
+                        .startObject("concrete-field")
+                            .field("type", "keyword")
+                        .endObject()
+                        .startObject("text-field")
+                            .field("type", "text")
+                            .field("copy_to", "alias-field")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder()
+            .startObject()
+                .field("text-field", "value")
+            .endObject());
+        MapperParsingException exception = expectThrows(MapperParsingException.class,
+            () -> mapper.parse(SourceToParse.source("test", "type", "1", bytes, XContentType.JSON)));
+
+        assertEquals("Cannot copy to a field alias [alias-field].", exception.getCause().getMessage());
+    }
+
+    public void testDynamicDottedFieldNameWithFieldAlias() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                            .field("path", "concrete-field")
+                        .endObject()
+                        .startObject("concrete-field")
+                            .field("type", "keyword")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("alias-field.dynamic-field")
+                    .field("type", "keyword")
+                .endObject()
+            .endObject());
+        MapperParsingException exception = expectThrows(MapperParsingException.class,
+            () -> mapper.parse(SourceToParse.source("test", "type", "1", bytes, XContentType.JSON)));
+
+        assertEquals("Could not dynamically add mapping for field [alias-field.dynamic-field]. "
+            + "Existing mapping for [alias-field] must be of type object but found [alias].", exception.getMessage());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
index c5032090092..b7ee74fb773 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleIndexingDocTests.java
@@ -45,7 +45,9 @@ public class DoubleIndexingDocTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
         IndexService index = createIndex("test");
         client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping, XContentType.JSON).get();
-        DocumentMapper mapper = index.mapperService().documentMapper("type");
+        MapperService mapperService = index.mapperService();
+        DocumentMapper mapper = mapperService.documentMapper();
+
         QueryShardContext context = index.newQueryShardContext(0, null, () -> 0L, null);
 
         ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", BytesReference
@@ -61,7 +63,6 @@ public class DoubleIndexingDocTests extends ESSingleNodeTestCase {
         assertNotNull(doc.dynamicMappingsUpdate());
         client().admin().indices().preparePutMapping("test").setType("type")
             .setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
-        mapper = index.mapperService().documentMapper("type");
 
         writer.addDocument(doc.rootDoc());
         writer.addDocument(doc.rootDoc());
@@ -69,25 +70,25 @@ public class DoubleIndexingDocTests extends ESSingleNodeTestCase {
         IndexReader reader = DirectoryReader.open(writer);
         IndexSearcher searcher = new IndexSearcher(reader);
 
-        TopDocs topDocs = searcher.search(mapper.mappers().getMapper("field1").fieldType().termQuery("value1", context), 10);
+        TopDocs topDocs = searcher.search(mapperService.fullName("field1").termQuery("value1", context), 10);
         assertThat(topDocs.totalHits, equalTo(2L));
 
-        topDocs = searcher.search(mapper.mappers().getMapper("field2").fieldType().termQuery("1", context), 10);
+        topDocs = searcher.search(mapperService.fullName("field2").termQuery("1", context), 10);
         assertThat(topDocs.totalHits, equalTo(2L));
 
-        topDocs = searcher.search(mapper.mappers().getMapper("field3").fieldType().termQuery("1.1", context), 10);
+        topDocs = searcher.search(mapperService.fullName("field3").termQuery("1.1", context), 10);
         assertThat(topDocs.totalHits, equalTo(2L));
 
-        topDocs = searcher.search(mapper.mappers().getMapper("field4").fieldType().termQuery("2010-01-01", context), 10);
+        topDocs = searcher.search(mapperService.fullName("field4").termQuery("2010-01-01", context), 10);
         assertThat(topDocs.totalHits, equalTo(2L));
 
-        topDocs = searcher.search(mapper.mappers().getMapper("field5").fieldType().termQuery("1", context), 10);
+        topDocs = searcher.search(mapperService.fullName("field5").termQuery("1", context), 10);
         assertThat(topDocs.totalHits, equalTo(2L));
 
-        topDocs = searcher.search(mapper.mappers().getMapper("field5").fieldType().termQuery("2", context), 10);
+        topDocs = searcher.search(mapperService.fullName("field5").termQuery("2", context), 10);
         assertThat(topDocs.totalHits, equalTo(2L));
 
-        topDocs = searcher.search(mapper.mappers().getMapper("field5").fieldType().termQuery("3", context), 10);
+        topDocs = searcher.search(mapperService.fullName("field5").termQuery("3", context), 10);
         assertThat(topDocs.totalHits, equalTo(2L));
         writer.close();
         reader.close();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index 0d7dde415aa..7d022b55454 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -625,11 +625,11 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
             .setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
 
         defaultMapper = index.mapperService().documentMapper("type");
-        FieldMapper mapper = defaultMapper.mappers().getMapper("s_long");
-        assertThat(mapper.fieldType().typeName(), equalTo("long"));
+        Mapper mapper = defaultMapper.mappers().getMapper("s_long");
+        assertThat(mapper.typeName(), equalTo("long"));
 
         mapper = defaultMapper.mappers().getMapper("s_double");
-        assertThat(mapper.fieldType().typeName(), equalTo("float"));
+        assertThat(mapper.typeName(), equalTo("float"));
     }
 
     public void testNumericDetectionDefault() throws Exception {
@@ -652,7 +652,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
             .setSource(doc.dynamicMappingsUpdate().toString(), XContentType.JSON).get());
 
         defaultMapper = index.mapperService().documentMapper("type");
-        FieldMapper mapper = defaultMapper.mappers().getMapper("s_long");
+        Mapper mapper = defaultMapper.mappers().getMapper("s_long");
         assertThat(mapper, instanceOf(TextFieldMapper.class));
 
         mapper = defaultMapper.mappers().getMapper("s_double");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
index d8e8c8e0e3d..62c764e8060 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
@@ -30,11 +30,11 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.hamcrest.Matchers;
 
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class DynamicTemplatesTests extends ESSingleNodeTestCase {
     public void testMatchTypeOnly() throws Exception {
@@ -45,7 +45,9 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
                 .endObject().endObject().endArray().endObject().endObject();
         IndexService index = createIndex("test");
         client().admin().indices().preparePutMapping("test").setType("person").setSource(builder).get();
-        DocumentMapper docMapper = index.mapperService().documentMapper("person");
+
+        MapperService mapperService = index.mapperService();
+        DocumentMapper docMapper = mapperService.documentMapper("person");
         builder = JsonXContent.contentBuilder();
         builder.startObject().field("s", "hello").field("l", 1).endObject();
         ParsedDocument parsedDoc = docMapper.parse(SourceToParse.source("test", "person", "1", BytesReference.bytes(builder),
@@ -53,14 +55,11 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
         client().admin().indices().preparePutMapping("test").setType("person")
             .setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
 
-        docMapper = index.mapperService().documentMapper("person");
-        DocumentFieldMappers mappers = docMapper.mappers();
+        assertThat(mapperService.fullName("s"), notNullValue());
+        assertEquals(IndexOptions.NONE, mapperService.fullName("s").indexOptions());
 
-        assertThat(mappers.getMapper("s"), Matchers.notNullValue());
-        assertEquals(IndexOptions.NONE, mappers.getMapper("s").fieldType().indexOptions());
-
-        assertThat(mappers.getMapper("l"), Matchers.notNullValue());
-        assertNotSame(IndexOptions.NONE, mappers.getMapper("l").fieldType().indexOptions());
+        assertThat(mapperService.fullName("l"), notNullValue());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("l").indexOptions());
 
 
     }
@@ -84,7 +83,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
         assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
         assertThat(f.fieldType().tokenized(), equalTo(false));
 
-        FieldMapper fieldMapper = docMapper.mappers().getMapper("name");
+        Mapper fieldMapper = docMapper.mappers().getMapper("name");
         assertNotNull(fieldMapper);
 
         f = doc.getField("multi1");
@@ -143,7 +142,7 @@ public class DynamicTemplatesTests extends ESSingleNodeTestCase {
         assertNotSame(IndexOptions.NONE, f.fieldType().indexOptions());
         assertThat(f.fieldType().tokenized(), equalTo(false));
 
-        FieldMapper fieldMapper = docMapper.mappers().getMapper("name");
+        Mapper fieldMapper = docMapper.mappers().getMapper("name");
         assertNotNull(fieldMapper);
 
         f = doc.getField("multi1");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java
new file mode 100644
index 00000000000..9f87ad3d039
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperTests.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class FieldAliasMapperTests extends ESSingleNodeTestCase {
+    private MapperService mapperService;
+    private DocumentMapperParser parser;
+
+    @Before
+    public void setup() {
+        IndexService indexService = createIndex("test");
+        mapperService = indexService.mapperService();
+        parser = mapperService.documentMapperParser();
+    }
+
+    public void testParsing() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                            .field("path", "concrete-field")
+                        .endObject()
+                        .startObject("concrete-field")
+                            .field("type", "keyword")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+    }
+
+    public void testParsingWithMissingPath() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        MapperParsingException exception = expectThrows(MapperParsingException.class,
+            () -> parser.parse("type", new CompressedXContent(mapping)));
+        assertEquals("The [path] property must be specified for field [alias-field].", exception.getMessage());
+    }
+
+    public void testParsingWithExtraArgument() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                            .field("path", "concrete-field")
+                            .field("extra-field", "extra-value")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        MapperParsingException exception = expectThrows(MapperParsingException.class,
+            () -> parser.parse("type", new CompressedXContent(mapping)));
+        assertEquals("Mapping definition for [alias-field] has unsupported parameters:  [extra-field : extra-value]",
+            exception.getMessage());
+    }
+
+    public void testMerge() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("first-field")
+                            .field("type", "keyword")
+                        .endObject()
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                            .field("path", "first-field")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
+
+        MappedFieldType firstFieldType = mapperService.fullName("alias-field");
+        assertEquals("first-field", firstFieldType.name());
+        assertTrue(firstFieldType instanceof KeywordFieldMapper.KeywordFieldType);
+
+        String newMapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+               .startObject("type")
+                    .startObject("properties")
+                        .startObject("second-field")
+                            .field("type", "text")
+                        .endObject()
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                            .field("path", "second-field")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        mapperService.merge("type", new CompressedXContent(newMapping), MergeReason.MAPPING_UPDATE);
+
+        MappedFieldType secondFieldType = mapperService.fullName("alias-field");
+        assertEquals("second-field", secondFieldType.name());
+        assertTrue(secondFieldType instanceof TextFieldMapper.TextFieldType);
+    }
+
+    public void testMergeFailure() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("concrete-field")
+                            .field("type", "text")
+                        .endObject()
+                        .startObject("alias-field")
+                            .field("type", "alias")
+                            .field("path", "concrete-field")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
+
+        String newMapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+               .startObject("type")
+                    .startObject("properties")
+                        .startObject("alias-field")
+                            .field("type", "keyword")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> mapperService.merge("type", new CompressedXContent(newMapping), MergeReason.MAPPING_UPDATE));
+        assertEquals("Cannot merge a field alias mapping [alias-field] with a mapping that is not for a field alias.",
+            exception.getMessage());
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
index 4f1b908cae8..6e27823f8a0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
@@ -28,10 +28,11 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
+
 public class FieldTypeLookupTests extends ESTestCase {
 
     public void testEmpty() {
@@ -48,7 +49,7 @@ public class FieldTypeLookupTests extends ESTestCase {
     public void testDefaultMapping() {
         FieldTypeLookup lookup = new FieldTypeLookup();
         try {
-            lookup.copyAndAddAll(MapperService.DEFAULT_MAPPING, Collections.emptyList());
+            lookup.copyAndAddAll(MapperService.DEFAULT_MAPPING, emptyList(), emptyList());
             fail();
         } catch (IllegalArgumentException expected) {
             assertEquals("Default mappings should not be added to the lookup", expected.getMessage());
@@ -58,7 +59,7 @@ public class FieldTypeLookupTests extends ESTestCase {
     public void testAddNewField() {
         FieldTypeLookup lookup = new FieldTypeLookup();
         MockFieldMapper f = new MockFieldMapper("foo");
-        FieldTypeLookup lookup2 = lookup.copyAndAddAll("type", newList(f));
+        FieldTypeLookup lookup2 = lookup.copyAndAddAll("type", newList(f), emptyList());
         assertNull(lookup.get("foo"));
         assertNull(lookup.get("bar"));
         assertEquals(f.fieldType(), lookup2.get("foo"));
@@ -70,68 +71,203 @@ public class FieldTypeLookupTests extends ESTestCase {
         MockFieldMapper f = new MockFieldMapper("foo");
         MockFieldMapper f2 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
-        lookup = lookup.copyAndAddAll("type1", newList(f));
-        FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2));
+        lookup = lookup.copyAndAddAll("type1", newList(f), emptyList());
+        FieldTypeLookup lookup2 = lookup.copyAndAddAll("type2", newList(f2), emptyList());
 
         assertEquals(1, size(lookup2.iterator()));
         assertSame(f.fieldType(), lookup2.get("foo"));
         assertEquals(f2.fieldType(), lookup2.get("foo"));
     }
 
-    public void testCheckCompatibilityMismatchedTypes() {
+    public void testMismatchedFieldTypes() {
         FieldMapper f1 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
-        lookup = lookup.copyAndAddAll("type", newList(f1));
+        lookup = lookup.copyAndAddAll("type", newList(f1), emptyList());
 
         OtherFakeFieldType ft2 = new OtherFakeFieldType();
         ft2.setName("foo");
         FieldMapper f2 = new MockFieldMapper("foo", ft2);
         try {
-            lookup.copyAndAddAll("type2", newList(f2));
+            lookup.copyAndAddAll("type2", newList(f2), emptyList());
             fail("expected type mismatch");
         } catch (IllegalArgumentException e) {
             assertTrue(e.getMessage().contains("cannot be changed from type [faketype] to [otherfaketype]"));
         }
     }
 
-    public void testCheckCompatibilityConflict() {
+    public void testConflictingFieldTypes() {
         FieldMapper f1 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
-        lookup = lookup.copyAndAddAll("type", newList(f1));
+        lookup = lookup.copyAndAddAll("type", newList(f1), emptyList());
 
         MappedFieldType ft2 = new MockFieldMapper.FakeFieldType();
         ft2.setName("foo");
         ft2.setBoost(2.0f);
         FieldMapper f2 = new MockFieldMapper("foo", ft2);
-        lookup.copyAndAddAll("type", newList(f2)); // boost is updateable, so ok since we are implicitly updating all types
-        lookup.copyAndAddAll("type2", newList(f2)); // boost is updateable, so ok if forcing
+        lookup.copyAndAddAll("type", newList(f2), emptyList()); // boost is updateable, so ok since we are implicitly updating all types
+        lookup.copyAndAddAll("type2", newList(f2), emptyList()); // boost is updateable, so ok if forcing
         // now with a non changeable setting
         MappedFieldType ft3 = new MockFieldMapper.FakeFieldType();
         ft3.setName("foo");
         ft3.setStored(true);
         FieldMapper f3 = new MockFieldMapper("foo", ft3);
         try {
-            lookup.copyAndAddAll("type2", newList(f3));
+            lookup.copyAndAddAll("type2", newList(f3), emptyList());
             fail("expected conflict");
         } catch (IllegalArgumentException e) {
             assertTrue(e.getMessage().contains("has different [store] values"));
         }
     }
 
-    public void testSimpleMatchFullNames() {
-        MockFieldMapper f1 = new MockFieldMapper("foo");
-        MockFieldMapper f2 = new MockFieldMapper("bar");
+    public void testAddFieldAlias() {
+        MockFieldMapper field = new MockFieldMapper("foo");
+        FieldAliasMapper alias = new FieldAliasMapper("alias", "alias", "foo");
+
         FieldTypeLookup lookup = new FieldTypeLookup();
-        lookup = lookup.copyAndAddAll("type", newList(f1, f2));
+        lookup = lookup.copyAndAddAll("type", newList(field), newList(alias));
+
+        MappedFieldType aliasType = lookup.get("alias");
+        assertEquals(field.fieldType(), aliasType);
+    }
+
+    public void testUpdateFieldAlias() {
+        // Add an alias 'alias' to the concrete field 'foo'.
+        MockFieldMapper.FakeFieldType fieldType1 = new MockFieldMapper.FakeFieldType();
+        MockFieldMapper field1 = new MockFieldMapper("foo", fieldType1);
+        FieldAliasMapper alias1 = new FieldAliasMapper("alias", "alias", "foo");
+
+        FieldTypeLookup lookup = new FieldTypeLookup();
+        lookup = lookup.copyAndAddAll("type", newList(field1), newList(alias1));
+
+        // Check that the alias refers to 'foo'.
+        MappedFieldType aliasType1 = lookup.get("alias");
+        assertEquals(fieldType1, aliasType1);
+
+        // Update the alias to refer to a new concrete field 'bar'.
+        MockFieldMapper.FakeFieldType fieldType2 = new MockFieldMapper.FakeFieldType();
+        fieldType2.setStored(!fieldType1.stored());
+        MockFieldMapper field2 = new MockFieldMapper("bar", fieldType2);
+
+        FieldAliasMapper alias2 = new FieldAliasMapper("alias", "alias", "bar");
+        lookup = lookup.copyAndAddAll("type", newList(field2), newList(alias2));
+
+        // Check that the alias now refers to 'bar'.
+        MappedFieldType aliasType2 = lookup.get("alias");
+        assertEquals(fieldType2, aliasType2);
+    }
+
+    public void testUpdateConcreteFieldWithAlias() {
+        // Add an alias 'alias' to the concrete field 'foo'.
+        FieldAliasMapper alias1 = new FieldAliasMapper("alias", "alias", "foo");
+        MockFieldMapper.FakeFieldType fieldType1 = new MockFieldMapper.FakeFieldType();
+        fieldType1.setBoost(1.0f);
+        MockFieldMapper field1 = new MockFieldMapper("foo", fieldType1);
+
+        FieldTypeLookup lookup = new FieldTypeLookup();
+        lookup = lookup.copyAndAddAll("type", newList(field1), newList(alias1));
+
+        // Check that the alias maps to this field type.
+        MappedFieldType aliasType1 = lookup.get("alias");
+        assertEquals(fieldType1, aliasType1);
+
+        // Update the boost for field 'foo'.
+        MockFieldMapper.FakeFieldType fieldType2 = new MockFieldMapper.FakeFieldType();
+        fieldType2.setBoost(2.0f);
+        MockFieldMapper field2 = new MockFieldMapper("foo", fieldType2);
+        lookup = lookup.copyAndAddAll("type", newList(field2), emptyList());
+
+        // Check that the alias maps to the new field type.
+        MappedFieldType aliasType2 = lookup.get("alias");
+        assertEquals(fieldType2, aliasType2);
+    }
+
+    public void testAliasThatRefersToAlias() {
+        MockFieldMapper field = new MockFieldMapper("foo");
+        FieldAliasMapper alias = new FieldAliasMapper("alias", "alias", "foo");
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", newList(field), newList(alias));
+
+        FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid-alias", "invalid-alias", "alias");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> lookup.copyAndAddAll("type", emptyList(), newList(invalidAlias)));
+        assertEquals("Invalid [path] value [alias] for field alias [invalid-alias]: an alias" +
+            " cannot refer to another alias.", e.getMessage());
+    }
+
+    public void testAliasThatRefersToItself() {
+        FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid-alias", "invalid-alias", "invalid-alias");
+
+        FieldTypeLookup lookup = new FieldTypeLookup();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> lookup.copyAndAddAll("type", emptyList(), newList(invalidAlias)));
+        assertEquals("Invalid [path] value [invalid-alias] for field alias [invalid-alias]: an alias" +
+            " cannot refer to itself.", e.getMessage());
+    }
+
+    public void testAliasWithNonExistentPath() {
+        FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid-alias", "invalid-alias", "non-existent");
+
+        FieldTypeLookup lookup = new FieldTypeLookup();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> lookup.copyAndAddAll("type", emptyList(), newList(invalidAlias)));
+        assertEquals("Invalid [path] value [non-existent] for field alias [invalid-alias]: an alias" +
+            " must refer to an existing field in the mappings.", e.getMessage());
+    }
+
+    public void testAddAliasWithPreexistingField() {
+        MockFieldMapper field = new MockFieldMapper("field");
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", newList(field), emptyList());
+
+        MockFieldMapper invalidField = new MockFieldMapper("invalid");
+        FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid", "invalid", "field");
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> lookup.copyAndAddAll("type", newList(invalidField), newList(invalidAlias)));
+        assertEquals("The name for field alias [invalid] has already been used to define a concrete field.",
+            e.getMessage());
+    }
+
+    public void testAddFieldWithPreexistingAlias() {
+        MockFieldMapper field = new MockFieldMapper("field");
+        FieldAliasMapper invalidAlias = new FieldAliasMapper("invalid", "invalid", "field");
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", newList(field), newList(invalidAlias));
+
+        MockFieldMapper invalidField = new MockFieldMapper("invalid");
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> lookup.copyAndAddAll("type", newList(invalidField), emptyList()));
+        assertEquals("The name for field [invalid] has already been used to define a field alias.",
+            e.getMessage());
+    }
+
+    public void testSimpleMatchToFullName() {
+        MockFieldMapper field1 = new MockFieldMapper("foo");
+        MockFieldMapper field2 = new MockFieldMapper("bar");
+
+        FieldAliasMapper alias1 = new FieldAliasMapper("food", "food", "foo");
+        FieldAliasMapper alias2 = new FieldAliasMapper("barometer", "barometer", "bar");
+
+        FieldTypeLookup lookup = new FieldTypeLookup();
+        lookup = lookup.copyAndAddAll("type",
+            newList(field1, field2),
+            newList(alias1, alias2));
+
         Collection<String> names = lookup.simpleMatchToFullName("b*");
+
         assertFalse(names.contains("foo"));
+        assertFalse(names.contains("food"));
+
         assertTrue(names.contains("bar"));
+        assertTrue(names.contains("barometer"));
     }
 
     public void testIteratorImmutable() {
         MockFieldMapper f1 = new MockFieldMapper("foo");
         FieldTypeLookup lookup = new FieldTypeLookup();
-        lookup = lookup.copyAndAddAll("type", newList(f1));
+        lookup = lookup.copyAndAddAll("type", newList(f1), emptyList());
 
         try {
             Iterator<MappedFieldType> itr = lookup.iterator();
@@ -144,7 +280,11 @@ public class FieldTypeLookupTests extends ESTestCase {
         }
     }
 
-    static List<FieldMapper> newList(FieldMapper... mapper) {
+    private static List<FieldMapper> newList(FieldMapper... mapper) {
+        return Arrays.asList(mapper);
+    }
+
+    private static List<FieldAliasMapper> newList(FieldAliasMapper... mapper) {
         return Arrays.asList(mapper);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GenericStoreDynamicTemplateTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GenericStoreDynamicTemplateTests.java
index 57a6173bc65..6999e39b70a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GenericStoreDynamicTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GenericStoreDynamicTemplateTests.java
@@ -23,10 +23,7 @@ import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.ParseContext.Document;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
@@ -38,13 +35,14 @@ public class GenericStoreDynamicTemplateTests extends ESSingleNodeTestCase {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-mapping.json");
         IndexService index = createIndex("test");
         client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping, XContentType.JSON).get();
-        DocumentMapper docMapper = index.mapperService().documentMapper("person");
+
+        MapperService mapperService = index.mapperService();
+
         byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-data.json");
-        ParsedDocument parsedDoc = docMapper.parse(SourceToParse.source("test", "person", "1", new BytesArray(json),
-                XContentType.JSON));
+        ParsedDocument parsedDoc = mapperService.documentMapper().parse(
+            SourceToParse.source("test", "person", "1", new BytesArray(json), XContentType.JSON));
         client().admin().indices().preparePutMapping("test").setType("person")
             .setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
-        docMapper = index.mapperService().documentMapper("person");
         Document doc = parsedDoc.rootDoc();
 
         IndexableField f = doc.getField("name");
@@ -52,8 +50,8 @@ public class GenericStoreDynamicTemplateTests extends ESSingleNodeTestCase {
         assertThat(f.stringValue(), equalTo("some name"));
         assertThat(f.fieldType().stored(), equalTo(true));
 
-        FieldMapper fieldMapper = docMapper.mappers().getMapper("name");
-        assertThat(fieldMapper.fieldType().stored(), equalTo(true));
+        MappedFieldType fieldType = mapperService.fullName("name");
+        assertThat(fieldType.stored(), equalTo(true));
 
         boolean stored = false;
         for (IndexableField field : doc.getFields("age")) {
@@ -61,7 +59,7 @@ public class GenericStoreDynamicTemplateTests extends ESSingleNodeTestCase {
         }
         assertTrue(stored);
 
-        fieldMapper = docMapper.mappers().getMapper("age");
-        assertThat(fieldMapper.fieldType().stored(), equalTo(true));
+        fieldType = mapperService.fullName("age");
+        assertThat(fieldType.stored(), equalTo(true));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
index facafaf180e..eabf0a849fa 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
@@ -287,7 +287,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
             .parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoPointFieldMapper.class));
 
         boolean ignoreZValue = ((GeoPointFieldMapper)fieldMapper).ignoreZValue().value();
@@ -364,10 +364,10 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
             .parse("type", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoPointFieldMapper.class));
 
-        Object nullValue = fieldMapper.fieldType().nullValue();
+        Object nullValue = ((GeoPointFieldMapper) fieldMapper).fieldType().nullValue();
         assertThat(nullValue, equalTo(new GeoPoint(1, 2)));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
index 7ff8c28f6dc..4c947a44a0a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
@@ -59,7 +59,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -83,7 +83,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         ShapeBuilder.Orientation orientation = ((GeoShapeFieldMapper)fieldMapper).fieldType().orientation();
@@ -121,7 +121,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         boolean coerce = ((GeoShapeFieldMapper)fieldMapper).coerce().value();
@@ -157,7 +157,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
             .parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         boolean ignoreZValue = ((GeoShapeFieldMapper)fieldMapper).ignoreZValue().value();
@@ -191,7 +191,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
             .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         Explicit<Boolean> ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
@@ -225,7 +225,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -248,7 +248,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -276,7 +276,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
 
 
             DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping));
-            FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+            Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
             assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
             GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -300,7 +300,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
 
 
             DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping));
-            FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+            Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
             assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
             GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -326,7 +326,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                     .endObject().endObject());
 
             DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping));
-            FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+            Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
             assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
             GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -350,7 +350,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                     .endObject().endObject());
 
             DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping));
-            FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+            Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
             assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
             GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -373,7 +373,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                     .endObject().endObject());
 
             DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping));
-            FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+            Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
             assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
             GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -395,7 +395,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -418,7 +418,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
 
 
             DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping));
-            FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+            Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
             assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
             GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -440,7 +440,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
                     .endObject().endObject());
 
             DocumentMapper defaultMapper = parser.parse("type1", new CompressedXContent(mapping));
-            FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+            Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
             assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
             GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -475,7 +475,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
         }
 
         // verify nothing changed
-        FieldMapper fieldMapper = docMapper.mappers().getMapper("shape");
+        Mapper fieldMapper = docMapper.mappers().getMapper("shape");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
@@ -600,7 +600,7 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase {
             .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
         GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JavaMultiFieldMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JavaMultiFieldMergeTests.java
index 32084c50310..8060c0a3f92 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/JavaMultiFieldMergeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/JavaMultiFieldMergeTests.java
@@ -38,69 +38,68 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/test-mapping1.json");
         MapperService mapperService = createIndex("test").mapperService();
 
-        DocumentMapper docMapper = mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), nullValue());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
+        assertThat(mapperService.fullName("name.indexed"), nullValue());
 
         BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("name", "some name").endObject());
-        Document doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
+        Document doc = mapperService.documentMapper().parse(
+            SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
         IndexableField f = doc.getField("name");
         assertThat(f, notNullValue());
         f = doc.getField("name.indexed");
         assertThat(f, nullValue());
 
         mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/test-mapping2.json");
-        docMapper = mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed2"), nullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed3"), nullValue());
+        assertThat(mapperService.fullName("name.indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed2"), nullValue());
+        assertThat(mapperService.fullName("name.not_indexed3"), nullValue());
 
-        doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
+        doc = mapperService.documentMapper().parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
         f = doc.getField("name");
         assertThat(f, notNullValue());
         f = doc.getField("name.indexed");
         assertThat(f, notNullValue());
 
         mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/test-mapping3.json");
-        docMapper = mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed2"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed3"), nullValue());
+        assertThat(mapperService.fullName("name.indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed2"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed3"), nullValue());
 
         mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/test-mapping4.json");
-        docMapper = mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed2"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed3"), notNullValue());
+        assertThat(mapperService.fullName("name.indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed2"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed3"), notNullValue());
     }
 
     public void testUpgradeFromMultiFieldTypeToMultiFields() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/test-mapping1.json");
         MapperService mapperService = createIndex("test").mapperService();
 
-        DocumentMapper docMapper = mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), nullValue());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
+        assertThat(mapperService.fullName("name.indexed"), nullValue());
 
         BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("name", "some name").endObject());
-        Document doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
+        Document doc = mapperService.documentMapper().parse(
+            SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
         IndexableField f = doc.getField("name");
         assertThat(f, notNullValue());
         f = doc.getField("name.indexed");
@@ -108,32 +107,31 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase {
 
 
         mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade1.json");
-        docMapper = mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed2"), nullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed3"), nullValue());
+        assertThat(mapperService.fullName("name.indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed2"), nullValue());
+        assertThat(mapperService.fullName("name.not_indexed3"), nullValue());
 
-        doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
+        doc = mapperService.documentMapper().parse(
+            SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
         f = doc.getField("name");
         assertThat(f, notNullValue());
         f = doc.getField("name.indexed");
         assertThat(f, notNullValue());
 
         mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade2.json");
-        docMapper = mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        mapperService.merge("person", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
 
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed2"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed3"), nullValue());
+        assertThat(mapperService.fullName("name.indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed2"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed3"), nullValue());
 
 
         mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multifield/merge/upgrade3.json");
@@ -146,10 +144,10 @@ public class JavaMultiFieldMergeTests extends ESSingleNodeTestCase {
         }
 
         // There are conflicts, so the `name.not_indexed3` has not been added
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed2"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed3"), nullValue());
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
+        assertThat(mapperService.fullName("name.indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed2"), notNullValue());
+        assertThat(mapperService.fullName("name.not_indexed3"), nullValue());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperMergeValidatorTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperMergeValidatorTests.java
new file mode 100644
index 00000000000..af17918baac
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperMergeValidatorTests.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
+
+public class MapperMergeValidatorTests extends ESTestCase {
+
+    public void testDuplicateFieldAliasAndObject() {
+        ObjectMapper objectMapper = createObjectMapper("some.path");
+        FieldAliasMapper aliasMapper = new FieldAliasMapper("path", "some.path", "field");
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            MapperMergeValidator.validateMapperStructure("type",
+                singletonList(objectMapper),
+                emptyList(),
+                singletonList(aliasMapper),
+                emptyMap(),
+                new FieldTypeLookup()));
+        assertEquals("Field [some.path] is defined both as an object and a field in [type]", e.getMessage());
+    }
+
+    public void testFieldAliasWithNestedScope() {
+        ObjectMapper objectMapper = createNestedObjectMapper("nested");
+        FieldAliasMapper aliasMapper = new FieldAliasMapper("alias", "nested.alias", "nested.field");
+
+        MapperMergeValidator.validateFieldReferences(emptyList(),
+            singletonList(aliasMapper),
+            Collections.singletonMap("nested", objectMapper),
+            new FieldTypeLookup());
+    }
+
+    public void testFieldAliasWithDifferentObjectScopes() {
+        Map<String, ObjectMapper> fullPathObjectMappers = new HashMap<>();
+        fullPathObjectMappers.put("object1", createObjectMapper("object1"));
+        fullPathObjectMappers.put("object2", createObjectMapper("object2"));
+
+        FieldAliasMapper aliasMapper = new FieldAliasMapper("alias", "object2.alias", "object1.field");
+
+        MapperMergeValidator.validateFieldReferences(emptyList(),
+            singletonList(aliasMapper),
+            fullPathObjectMappers,
+            new FieldTypeLookup());
+    }
+
+    public void testFieldAliasWithNestedTarget() {
+        ObjectMapper objectMapper = createNestedObjectMapper("nested");
+        FieldAliasMapper aliasMapper = new FieldAliasMapper("alias", "alias", "nested.field");
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            MapperMergeValidator.validateFieldReferences(emptyList(),
+                singletonList(aliasMapper),
+                Collections.singletonMap("nested", objectMapper),
+                new FieldTypeLookup()));
+
+        String expectedMessage = "Invalid [path] value [nested.field] for field alias [alias]: " +
+            "an alias must have the same nested scope as its target. The alias is not nested, " +
+            "but the target's nested scope is [nested].";
+        assertEquals(expectedMessage, e.getMessage());
+    }
+
+    public void testFieldAliasWithDifferentNestedScopes() {
+        Map<String, ObjectMapper> fullPathObjectMappers = new HashMap<>();
+        fullPathObjectMappers.put("nested1", createNestedObjectMapper("nested1"));
+        fullPathObjectMappers.put("nested2", createNestedObjectMapper("nested2"));
+
+        FieldAliasMapper aliasMapper = new FieldAliasMapper("alias", "nested2.alias", "nested1.field");
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            MapperMergeValidator.validateFieldReferences(emptyList(),
+                singletonList(aliasMapper),
+                fullPathObjectMappers,
+                new FieldTypeLookup()));
+
+
+        String expectedMessage = "Invalid [path] value [nested1.field] for field alias [nested2.alias]: " +
+            "an alias must have the same nested scope as its target. The alias's nested scope is [nested2], " +
+            "but the target's nested scope is [nested1].";
+        assertEquals(expectedMessage, e.getMessage());
+    }
+
+    private static ObjectMapper createObjectMapper(String name) {
+        return new ObjectMapper(name, name, true,
+            ObjectMapper.Nested.NO,
+            ObjectMapper.Dynamic.FALSE, emptyMap(), Settings.EMPTY);
+    }
+
+    private static ObjectMapper createNestedObjectMapper(String name) {
+        return new ObjectMapper(name, name, true,
+            ObjectMapper.Nested.newNested(false, false),
+            ObjectMapper.Dynamic.FALSE, emptyMap(), Settings.EMPTY);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index 6bccb7106f6..20e0dd4639c 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -235,6 +235,41 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
             containsString("cannot have nested fields when index sort is activated"));
     }
 
+     public void testFieldAliasWithMismatchedNestedScope() throws Throwable {
+        IndexService indexService = createIndex("test");
+        MapperService mapperService = indexService.mapperService();
+
+        CompressedXContent mapping = new CompressedXContent(BytesReference.bytes(
+            XContentFactory.jsonBuilder().startObject()
+                .startObject("properties")
+                    .startObject("nested")
+                        .field("type", "nested")
+                        .startObject("properties")
+                            .startObject("field")
+                                .field("type", "text")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()));
+
+        mapperService.merge("type", mapping, MergeReason.MAPPING_UPDATE);
+
+        CompressedXContent mappingUpdate = new CompressedXContent(BytesReference.bytes(
+            XContentFactory.jsonBuilder().startObject()
+                .startObject("properties")
+                    .startObject("alias")
+                        .field("type", "alias")
+                        .field("path", "nested.field")
+                    .endObject()
+                .endObject()
+            .endObject()));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> mapperService.merge("type", mappingUpdate, MergeReason.MAPPING_UPDATE));
+        assertThat(e.getMessage(), containsString("Invalid [path] value [nested.field] for field alias [alias]"));
+    }
+
     public void testForbidMultipleTypes() throws IOException {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
         MapperService mapperService = createIndex("test").mapperService();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
index 45bb8090206..0f3d5193c28 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
@@ -55,9 +56,15 @@ public class MultiFieldTests extends ESSingleNodeTestCase {
     }
 
     private void testMultiField(String mapping) throws Exception {
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
+        IndexService indexService = createIndex("test");
+        MapperService mapperService = indexService.mapperService();
+
+        indexService.mapperService().merge("person", new CompressedXContent(mapping),
+            MapperService.MergeReason.MAPPING_UPDATE);
+
         BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/multifield/test-data.json"));
-        Document doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
+        Document doc = mapperService.documentMapper().parse(
+            SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
 
         IndexableField f = doc.getField("name");
         assertThat(f.name(), equalTo("name"));
@@ -84,37 +91,37 @@ public class MultiFieldTests extends ESSingleNodeTestCase {
         assertThat(f.name(), equalTo("object1.multi1.string"));
         assertThat(f.binaryValue(), equalTo(new BytesRef("2010-01-01")));
 
-        assertThat(docMapper.mappers().getMapper("name"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name"), instanceOf(TextFieldMapper.class));
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name").fieldType().stored(), equalTo(true));
-        assertThat(docMapper.mappers().getMapper("name").fieldType().tokenized(), equalTo(true));
+        assertThat(mapperService.fullName("name"), notNullValue());
+        assertThat(mapperService.fullName("name"), instanceOf(TextFieldType.class));
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name").indexOptions());
+        assertThat(mapperService.fullName("name").stored(), equalTo(true));
+        assertThat(mapperService.fullName("name").tokenized(), equalTo(true));
 
-        assertThat(docMapper.mappers().getMapper("name.indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.indexed"), instanceOf(TextFieldMapper.class));
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name.indexed").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.indexed").fieldType().stored(), equalTo(false));
-        assertThat(docMapper.mappers().getMapper("name.indexed").fieldType().tokenized(), equalTo(true));
+        assertThat(mapperService.fullName("name.indexed"), notNullValue());
+        assertThat(mapperService.fullName("name"), instanceOf(TextFieldType.class));
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name.indexed").indexOptions());
+        assertThat(mapperService.fullName("name.indexed").stored(), equalTo(false));
+        assertThat(mapperService.fullName("name.indexed").tokenized(), equalTo(true));
 
-        assertThat(docMapper.mappers().getMapper("name.not_indexed"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed"), instanceOf(TextFieldMapper.class));
-        assertEquals(IndexOptions.NONE, docMapper.mappers().getMapper("name.not_indexed").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.not_indexed").fieldType().stored(), equalTo(true));
-        assertThat(docMapper.mappers().getMapper("name.not_indexed").fieldType().tokenized(), equalTo(true));
+        assertThat(mapperService.fullName("name.not_indexed"), notNullValue());
+        assertThat(mapperService.fullName("name"), instanceOf(TextFieldType.class));
+        assertEquals(IndexOptions.NONE, mapperService.fullName("name.not_indexed").indexOptions());
+        assertThat(mapperService.fullName("name.not_indexed").stored(), equalTo(true));
+        assertThat(mapperService.fullName("name.not_indexed").tokenized(), equalTo(true));
 
-        assertThat(docMapper.mappers().getMapper("name.test1"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("name.test1"), instanceOf(TextFieldMapper.class));
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("name.test1").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("name.test1").fieldType().stored(), equalTo(true));
-        assertThat(docMapper.mappers().getMapper("name.test1").fieldType().tokenized(), equalTo(true));
-        assertThat(docMapper.mappers().getMapper("name.test1").fieldType().eagerGlobalOrdinals(), equalTo(true));
+        assertThat(mapperService.fullName("name.test1"), notNullValue());
+        assertThat(mapperService.fullName("name"), instanceOf(TextFieldType.class));
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("name.test1").indexOptions());
+        assertThat(mapperService.fullName("name.test1").stored(), equalTo(true));
+        assertThat(mapperService.fullName("name.test1").tokenized(), equalTo(true));
+        assertThat(mapperService.fullName("name.test1").eagerGlobalOrdinals(), equalTo(true));
 
-        assertThat(docMapper.mappers().getMapper("object1.multi1"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("object1.multi1"), instanceOf(DateFieldMapper.class));
-        assertThat(docMapper.mappers().getMapper("object1.multi1.string"), notNullValue());
-        assertThat(docMapper.mappers().getMapper("object1.multi1.string"), instanceOf(KeywordFieldMapper.class));
-        assertNotSame(IndexOptions.NONE, docMapper.mappers().getMapper("object1.multi1.string").fieldType().indexOptions());
-        assertThat(docMapper.mappers().getMapper("object1.multi1.string").fieldType().tokenized(), equalTo(false));
+        assertThat(mapperService.fullName("object1.multi1"), notNullValue());
+        assertThat(mapperService.fullName("object1.multi1"), instanceOf(DateFieldMapper.DateFieldType.class));
+        assertThat(mapperService.fullName("object1.multi1.string"), notNullValue());
+        assertThat(mapperService.fullName("object1.multi1.string"), instanceOf(KeywordFieldMapper.KeywordFieldType.class));
+        assertNotSame(IndexOptions.NONE, mapperService.fullName("object1.multi1.string").indexOptions());
+        assertThat(mapperService.fullName("object1.multi1.string").tokenized(), equalTo(false));
     }
 
     public void testBuildThenParse() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java
index 271501281cd..6bb15432b1f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PathMatchDynamicTemplateTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PathMatchDynamicTemplateTests.java
index 3ad53513c51..9546fb5136e 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/PathMatchDynamicTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/PathMatchDynamicTemplateTests.java
@@ -23,10 +23,7 @@ import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.ParseContext.Document;
-import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
@@ -38,13 +35,14 @@ public class PathMatchDynamicTemplateTests extends ESSingleNodeTestCase {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/pathmatch/test-mapping.json");
         IndexService index = createIndex("test");
         client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping, XContentType.JSON).get();
-        DocumentMapper docMapper = index.mapperService().documentMapper("person");
+
+        MapperService mapperService = index.mapperService();
+
         byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/dynamictemplate/pathmatch/test-data.json");
-        ParsedDocument parsedDoc = docMapper.parse(SourceToParse.source("test", "person", "1", new BytesArray(json),
-                XContentType.JSON));
+        ParsedDocument parsedDoc = mapperService.documentMapper().parse(
+            SourceToParse.source("test", "person", "1", new BytesArray(json), XContentType.JSON));
         client().admin().indices().preparePutMapping("test").setType("person")
             .setSource(parsedDoc.dynamicMappingsUpdate().toString(), XContentType.JSON).get();
-        docMapper = index.mapperService().documentMapper("person");
         Document doc = parsedDoc.rootDoc();
 
         IndexableField f = doc.getField("name");
@@ -52,26 +50,26 @@ public class PathMatchDynamicTemplateTests extends ESSingleNodeTestCase {
         assertThat(f.stringValue(), equalTo("top_level"));
         assertThat(f.fieldType().stored(), equalTo(false));
 
-        FieldMapper fieldMapper = docMapper.mappers().getMapper("name");
-        assertThat(fieldMapper.fieldType().stored(), equalTo(false));
+        MappedFieldType fieldType = mapperService.fullName("name");
+        assertThat(fieldType.stored(), equalTo(false));
 
         f = doc.getField("obj1.name");
         assertThat(f.name(), equalTo("obj1.name"));
         assertThat(f.fieldType().stored(), equalTo(true));
 
-        fieldMapper = docMapper.mappers().getMapper("obj1.name");
-        assertThat(fieldMapper.fieldType().stored(), equalTo(true));
+        fieldType = mapperService.fullName("obj1.name");
+        assertThat(fieldType.stored(), equalTo(true));
 
         f = doc.getField("obj1.obj2.name");
         assertThat(f.name(), equalTo("obj1.obj2.name"));
         assertThat(f.fieldType().stored(), equalTo(false));
 
-        fieldMapper = docMapper.mappers().getMapper("obj1.obj2.name");
-        assertThat(fieldMapper.fieldType().stored(), equalTo(false));
+        fieldType = mapperService.fullName("obj1.obj2.name");
+        assertThat(fieldType.stored(), equalTo(false));
 
         // verify more complex path_match expressions
 
-        fieldMapper = docMapper.mappers().getMapper("obj3.obj4.prop1");
-        assertNotNull(fieldMapper);
+        fieldType = mapperService.fullName("obj3.obj4.prop1");
+        assertNotNull(fieldType);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java
index 95ffc373e6b..a07192df804 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java
@@ -28,13 +28,14 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
-import java.util.Collections;
+import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -84,9 +85,11 @@ public class StoredNumericValuesTests extends ESSingleNodeTestCase {
         DirectoryReader reader = DirectoryReader.open(writer);
         IndexSearcher searcher = new IndexSearcher(reader);
 
-        CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(
-                Collections.emptySet(), Collections.singletonList("field*"), false);
+        Set<String> fieldNames = Sets.newHashSet("field1", "field2", "field3", "field4", "field5",
+            "field6", "field7", "field8", "field9", "field10");
+        CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(fieldNames, false);
         searcher.doc(0, fieldsVisitor);
+
         fieldsVisitor.postProcess(mapperService);
         assertThat(fieldsVisitor.fields().size(), equalTo(10));
         assertThat(fieldsVisitor.fields().get("field1").size(), equalTo(1));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
index e3dc8ff0b78..4736cbe4712 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
@@ -490,9 +490,10 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
-
         assertEquals(mapping, mapper.mappingSource().toString());
-        assertTrue(mapper.mappers().getMapper("field").fieldType().eagerGlobalOrdinals());
+
+        FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field");
+        assertTrue(fieldMapper.fieldType().eagerGlobalOrdinals());
     }
 
     public void testFielddata() throws IOException {
@@ -504,8 +505,10 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
 
         DocumentMapper disabledMapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, disabledMapper.mappingSource().toString());
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> disabledMapper.mappers().getMapper("field").fieldType().fielddataBuilder("test"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            FieldMapper fieldMapper = (FieldMapper) disabledMapper.mappers().getMapper("field");
+            fieldMapper.fieldType().fielddataBuilder("test");
+        });
         assertThat(e.getMessage(), containsString("Fielddata is disabled"));
 
         mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
@@ -518,7 +521,9 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
         DocumentMapper enabledMapper = parser.parse("type", new CompressedXContent(mapping));
 
         assertEquals(mapping, enabledMapper.mappingSource().toString());
-        enabledMapper.mappers().getMapper("field").fieldType().fielddataBuilder("test"); // no exception this time
+
+        FieldMapper enabledFieldMapper = (FieldMapper) enabledMapper.mappers().getMapper("field");
+        enabledFieldMapper.fieldType().fielddataBuilder("test"); // no exception this time
 
         String illegalMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field")
@@ -547,7 +552,9 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
         assertEquals(mapping, mapper.mappingSource().toString());
-        TextFieldType fieldType = (TextFieldType) mapper.mappers().getMapper("field").fieldType();
+        TextFieldMapper fieldMapper = (TextFieldMapper) mapper.mappers().getMapper("field");
+        TextFieldType fieldType = fieldMapper.fieldType();
+
         assertThat(fieldType.fielddataMinFrequency(), equalTo(2d));
         assertThat(fieldType.fielddataMaxFrequency(), equalTo((double) Integer.MAX_VALUE));
         assertThat(fieldType.fielddataMinSegmentSize(), equalTo(1000));
@@ -630,7 +637,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
 
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
-            FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
+            FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
             FieldType ft = prefix.fieldType;
             assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, ft.indexOptions());
         }
@@ -646,7 +653,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
 
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
-            FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
+            FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
             FieldType ft = prefix.fieldType;
             assertEquals(IndexOptions.DOCS, ft.indexOptions());
             assertFalse(ft.storeTermVectors());
@@ -663,7 +670,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
 
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
-            FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
+            FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
             FieldType ft = prefix.fieldType;
             if (indexService.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_4_0)) {
                 assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
@@ -684,7 +691,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
 
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
-            FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
+            FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
             FieldType ft = prefix.fieldType;
             if (indexService.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_4_0)) {
                 assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
@@ -705,7 +712,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
 
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
-            FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
+            FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
             FieldType ft = prefix.fieldType;
             if (indexService.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_4_0)) {
                 assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
@@ -836,10 +843,13 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
 
             assertThat(mapper.mappers().getMapper("field._index_prefix").toString(), containsString("prefixChars=1:10"));
 
-            Query q = mapper.mappers().getMapper("field").fieldType().prefixQuery("goin", CONSTANT_SCORE_REWRITE, queryShardContext);
+            FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field");
+            MappedFieldType fieldType = fieldMapper.fieldType;
+
+            Query q = fieldType.prefixQuery("goin", CONSTANT_SCORE_REWRITE, queryShardContext);
+
             assertEquals(new ConstantScoreQuery(new TermQuery(new Term("field._index_prefix", "goin"))), q);
-            q = mapper.mappers().getMapper("field").fieldType().prefixQuery("internationalisatio",
-                CONSTANT_SCORE_REWRITE, queryShardContext);
+            q = fieldType.prefixQuery("internationalisatio", CONSTANT_SCORE_REWRITE, queryShardContext);
             assertEquals(new PrefixQuery(new Term("field", "internationalisatio")), q);
 
             ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", BytesReference
@@ -864,17 +874,16 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
             CompressedXContent json = new CompressedXContent(mapping);
             DocumentMapper mapper = parser.parse("type", json);
 
-            Query q1 = mapper.mappers().getMapper("field").fieldType().prefixQuery("g",
-                CONSTANT_SCORE_REWRITE, queryShardContext);
+            FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field");
+            MappedFieldType fieldType = fieldMapper.fieldType;
+
+            Query q1 = fieldType.prefixQuery("g", CONSTANT_SCORE_REWRITE, queryShardContext);
             assertThat(q1, instanceOf(PrefixQuery.class));
-            Query q2 = mapper.mappers().getMapper("field").fieldType().prefixQuery("go",
-                CONSTANT_SCORE_REWRITE, queryShardContext);
+            Query q2 = fieldType.prefixQuery("go", CONSTANT_SCORE_REWRITE, queryShardContext);
             assertThat(q2, instanceOf(ConstantScoreQuery.class));
-            Query q5 = mapper.mappers().getMapper("field").fieldType().prefixQuery("going",
-                CONSTANT_SCORE_REWRITE, queryShardContext);
+            Query q5 = fieldType.prefixQuery("going", CONSTANT_SCORE_REWRITE, queryShardContext);
             assertThat(q5, instanceOf(ConstantScoreQuery.class));
-            Query q6 = mapper.mappers().getMapper("field").fieldType().prefixQuery("goings",
-                CONSTANT_SCORE_REWRITE, queryShardContext);
+            Query q6 = fieldType.prefixQuery("goings", CONSTANT_SCORE_REWRITE, queryShardContext);
             assertThat(q6, instanceOf(PrefixQuery.class));
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
index 5478e172fed..fe39345dadd 100644
--- a/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
@@ -27,6 +28,7 @@ import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
@@ -39,19 +41,16 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe
 
     @Override
     protected CommonTermsQueryBuilder doCreateTestQueryBuilder() {
-        CommonTermsQueryBuilder query;
-
         int numberOfTerms = randomIntBetween(0, 10);
         StringBuilder text = new StringBuilder("");
         for (int i = 0; i < numberOfTerms; i++) {
             text.append(randomAlphaOfLengthBetween(1, 10)).append(" ");
         }
-        // mapped or unmapped field
-        if (randomBoolean()) {
-            query = new CommonTermsQueryBuilder(STRING_FIELD_NAME, text.toString());
-        } else {
-            query = new CommonTermsQueryBuilder(randomAlphaOfLengthBetween(1, 10), text.toString());
-        }
+
+        String fieldName = randomFrom(STRING_FIELD_NAME,
+            STRING_ALIAS_FIELD_NAME,
+            randomAlphaOfLengthBetween(1, 10));
+        CommonTermsQueryBuilder query = new CommonTermsQueryBuilder(fieldName, text.toString());
 
         if (randomBoolean()) {
             query.cutoffFrequency(randomIntBetween(1, 10));
@@ -100,6 +99,14 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe
     protected void doAssertLuceneQuery(CommonTermsQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(ExtendedCommonTermsQuery.class));
         ExtendedCommonTermsQuery extendedCommonTermsQuery = (ExtendedCommonTermsQuery) query;
+
+        List<Term> terms = extendedCommonTermsQuery.getTerms();
+        if (!terms.isEmpty()) {
+            String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+            String actualFieldName = terms.iterator().next().field();
+            assertThat(actualFieldName, equalTo(expectedFieldName));
+        }
+
         assertThat(extendedCommonTermsQuery.getHighFreqMinimumNumberShouldMatchSpec(), equalTo(queryBuilder.highFreqMinimumShouldMatch()));
         assertThat(extendedCommonTermsQuery.getLowFreqMinimumNumberShouldMatchSpec(), equalTo(queryBuilder.lowFreqMinimumShouldMatch()));
     }
diff --git a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
index d4547eee26f..88742e08554 100644
--- a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
@@ -76,7 +76,7 @@ public class ExistsQueryBuilderTests extends AbstractQueryTestCase<ExistsQueryBu
             if (fields.size() == 1) {
                 assertThat(query, instanceOf(ConstantScoreQuery.class));
                 ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query;
-                String field = fields.iterator().next();
+                String field = expectedFieldName(fields.iterator().next());
                 assertThat(constantScoreQuery.getQuery(), instanceOf(TermQuery.class));
                 TermQuery termQuery = (TermQuery) constantScoreQuery.getQuery();
                 assertEquals(field, termQuery.getTerm().text());
@@ -99,7 +99,7 @@ public class ExistsQueryBuilderTests extends AbstractQueryTestCase<ExistsQueryBu
         } else if (fields.size() == 1) {
             assertThat(query, instanceOf(ConstantScoreQuery.class));
             ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query;
-            String field = fields.iterator().next();
+            String field = expectedFieldName(fields.iterator().next());
             if (context.getQueryShardContext().getObjectMapper(field) != null) {
                 assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class));
                 BooleanQuery booleanQuery = (BooleanQuery) constantScoreQuery.getQuery();
diff --git a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
index d7fe00e730c..9d98a12358f 100644
--- a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
@@ -45,11 +44,7 @@ public class FieldMaskingSpanQueryBuilderTests extends AbstractQueryTestCase<Fie
 
     @Override
     protected void doAssertLuceneQuery(FieldMaskingSpanQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
-        String fieldInQuery = queryBuilder.fieldName();
-        MappedFieldType fieldType = context.getQueryShardContext().fieldMapper(fieldInQuery);
-        if (fieldType != null) {
-            fieldInQuery = fieldType.name();
-        }
+        String fieldInQuery = expectedFieldName(queryBuilder.fieldName());
         assertThat(query, instanceOf(FieldMaskingSpanQuery.class));
         FieldMaskingSpanQuery fieldMaskingSpanQuery = (FieldMaskingSpanQuery) query;
         assertThat(fieldMaskingSpanQuery.getField(), equalTo(fieldInQuery));
diff --git a/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
index 4fae80d09a5..0a67ab50ae8 100644
--- a/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
@@ -41,7 +41,8 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
 
     @Override
     protected FuzzyQueryBuilder doCreateTestQueryBuilder() {
-        FuzzyQueryBuilder query = new FuzzyQueryBuilder(STRING_FIELD_NAME, getRandomValueForFieldName(STRING_FIELD_NAME));
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME);
+        FuzzyQueryBuilder query = new FuzzyQueryBuilder(fieldName, getRandomValueForFieldName(fieldName));
         if (randomBoolean()) {
             query.fuzziness(randomFuzziness(query.fieldName()));
         }
@@ -76,6 +77,11 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
     @Override
     protected void doAssertLuceneQuery(FuzzyQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(FuzzyQuery.class));
+
+        FuzzyQuery fuzzyQuery = (FuzzyQuery) query;
+        String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+        String actualFieldName = fuzzyQuery.getTerm().field();
+        assertThat(actualFieldName, equalTo(expectedFieldName));
     }
 
     public void testIllegalArguments() {
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
index d1f7972c7f4..d0d075af21a 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
@@ -39,6 +39,7 @@ import java.io.IOException;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBoundingBoxQueryBuilder> {
     /** Randomly generate either NaN or one of the two infinity values. */
@@ -46,7 +47,8 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
 
     @Override
     protected GeoBoundingBoxQueryBuilder doCreateTestQueryBuilder() {
-        GeoBoundingBoxQueryBuilder builder = new GeoBoundingBoxQueryBuilder(GEO_POINT_FIELD_NAME);
+        String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME);
+        GeoBoundingBoxQueryBuilder builder = new GeoBoundingBoxQueryBuilder(fieldName);
         Rectangle box = RandomShapeGenerator.xRandomRectangle(random(), RandomShapeGenerator.xRandomPoint(random()));
 
         if (randomBoolean()) {
@@ -117,7 +119,7 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
     public void testExceptionOnMissingTypes() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length == 0);
         QueryShardException e = expectThrows(QueryShardException.class, super::testToQuery);
-        assertEquals("failed to find geo_point field [mapped_geo_point]", e.getMessage());
+        assertThat(e.getMessage(), startsWith("failed to find geo_point field [mapped_geo_point"));
     }
 
     public void testBrokenCoordinateCannotBeSet() {
@@ -216,13 +218,14 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
             assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery);
         } else if (query instanceof IndexOrDocValuesQuery) { // TODO: remove the if statement once we always use LatLonPoint
             Query indexQuery = ((IndexOrDocValuesQuery) query).getIndexQuery();
-            assertEquals(LatLonPoint.newBoxQuery(queryBuilder.fieldName(),
+            String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+            assertEquals(LatLonPoint.newBoxQuery(expectedFieldName,
                     queryBuilder.bottomRight().lat(),
                     queryBuilder.topLeft().lat(),
                     queryBuilder.topLeft().lon(),
                     queryBuilder.bottomRight().lon()), indexQuery);
             Query dvQuery = ((IndexOrDocValuesQuery) query).getRandomAccessQuery();
-            assertEquals(LatLonDocValuesField.newSlowBoxQuery(queryBuilder.fieldName(),
+            assertEquals(LatLonDocValuesField.newSlowBoxQuery(expectedFieldName,
                     queryBuilder.bottomRight().lat(),
                     queryBuilder.topLeft().lat(),
                     queryBuilder.topLeft().lon(),
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
index eca99f95c48..7b7a1dcb7a5 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
@@ -43,7 +43,8 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
 
     @Override
     protected GeoDistanceQueryBuilder doCreateTestQueryBuilder() {
-        GeoDistanceQueryBuilder qb = new GeoDistanceQueryBuilder(GEO_POINT_FIELD_NAME);
+        String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME);
+        GeoDistanceQueryBuilder qb = new GeoDistanceQueryBuilder(fieldName);
         String distance = "" + randomDouble();
         if (randomBoolean()) {
             DistanceUnit unit = randomFrom(DistanceUnit.values());
@@ -130,13 +131,15 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
         // TODO: remove the if statement once we always use LatLonPoint
         if (query instanceof IndexOrDocValuesQuery) {
             Query indexQuery = ((IndexOrDocValuesQuery) query).getIndexQuery();
-            assertEquals(LatLonPoint.newDistanceQuery(queryBuilder.fieldName(),
+
+            String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+            assertEquals(LatLonPoint.newDistanceQuery(expectedFieldName,
                     queryBuilder.point().lat(),
                     queryBuilder.point().lon(),
                     queryBuilder.distance()),
                     indexQuery);
             Query dvQuery = ((IndexOrDocValuesQuery) query).getRandomAccessQuery();
-            assertEquals(LatLonDocValuesField.newSlowDistanceQuery(queryBuilder.fieldName(),
+            assertEquals(LatLonDocValuesField.newSlowDistanceQuery(expectedFieldName,
                     queryBuilder.point().lat(),
                     queryBuilder.point().lon(),
                     queryBuilder.distance()),
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
index 96716564366..8f1ae8eee07 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
@@ -44,8 +44,9 @@ import static org.hamcrest.CoreMatchers.notNullValue;
 public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygonQueryBuilder> {
     @Override
     protected GeoPolygonQueryBuilder doCreateTestQueryBuilder() {
+        String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME);
         List<GeoPoint> polygon = randomPolygon();
-        GeoPolygonQueryBuilder builder = new GeoPolygonQueryBuilder(GEO_POINT_FIELD_NAME, polygon);
+        GeoPolygonQueryBuilder builder = new GeoPolygonQueryBuilder(fieldName, polygon);
         if (randomBoolean()) {
             builder.setValidationMethod(randomFrom(GeoValidationMethod.values()));
         }
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
index e5da5d7f971..694b63b141b 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
@@ -42,13 +42,13 @@ import static org.hamcrest.Matchers.notNullValue;
 public class MatchPhrasePrefixQueryBuilderTests extends AbstractQueryTestCase<MatchPhrasePrefixQueryBuilder> {
     @Override
     protected MatchPhrasePrefixQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = randomFrom(STRING_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME,
-                DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME,
+            INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
         if (fieldName.equals(DATE_FIELD_NAME)) {
             assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         }
         Object value;
-        if (fieldName.equals(STRING_FIELD_NAME)) {
+        if (isTextField(fieldName)) {
             int terms = randomIntBetween(0, 3);
             StringBuilder builder = new StringBuilder();
             for (int i = 0; i < terms; i++) {
@@ -61,7 +61,7 @@ public class MatchPhrasePrefixQueryBuilderTests extends AbstractQueryTestCase<Ma
 
         MatchPhrasePrefixQueryBuilder matchQuery = new MatchPhrasePrefixQueryBuilder(fieldName, value);
 
-        if (randomBoolean() && fieldName.equals(STRING_FIELD_NAME)) {
+        if (randomBoolean() && isTextField(fieldName)) {
             matchQuery.analyzer(randomFrom("simple", "keyword", "whitespace"));
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
index fb03d54aeff..e59c5d6e0c4 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
@@ -45,13 +45,13 @@ import static org.hamcrest.Matchers.notNullValue;
 public class MatchPhraseQueryBuilderTests extends AbstractQueryTestCase<MatchPhraseQueryBuilder> {
     @Override
     protected MatchPhraseQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = randomFrom(STRING_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME,
-                DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME,
+            INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
         if (fieldName.equals(DATE_FIELD_NAME)) {
             assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         }
         Object value;
-        if (fieldName.equals(STRING_FIELD_NAME)) {
+        if (isTextField(fieldName)) {
             int terms = randomIntBetween(0, 3);
             StringBuilder builder = new StringBuilder();
             for (int i = 0; i < terms; i++) {
@@ -64,7 +64,7 @@ public class MatchPhraseQueryBuilderTests extends AbstractQueryTestCase<MatchPhr
 
         MatchPhraseQueryBuilder matchQuery = new MatchPhraseQueryBuilder(fieldName, value);
 
-        if (randomBoolean() && fieldName.equals(STRING_FIELD_NAME)) {
+        if (randomBoolean() && isTextField(fieldName)) {
             matchQuery.analyzer(randomFrom("simple", "keyword", "whitespace"));
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index 05a4b99f19d..7d72ad9a4e1 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -47,6 +48,7 @@ import org.hamcrest.Matcher;
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
@@ -59,13 +61,13 @@ import static org.hamcrest.Matchers.notNullValue;
 public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuilder> {
     @Override
     protected MatchQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = randomFrom(STRING_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME,
-                DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
+        String fieldName = STRING_ALIAS_FIELD_NAME; //randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME,
+            //INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
         if (fieldName.equals(DATE_FIELD_NAME)) {
             assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         }
         Object value;
-        if (fieldName.equals(STRING_FIELD_NAME)) {
+        if (isTextField(fieldName)) {
             int terms = randomIntBetween(0, 3);
             StringBuilder builder = new StringBuilder();
             for (int i = 0; i < terms; i++) {
@@ -79,11 +81,11 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
         MatchQueryBuilder matchQuery = new MatchQueryBuilder(fieldName, value);
         matchQuery.operator(randomFrom(Operator.values()));
 
-        if (randomBoolean() && fieldName.equals(STRING_FIELD_NAME)) {
+        if (randomBoolean() && isTextField(fieldName)) {
             matchQuery.analyzer(randomFrom("simple", "keyword", "whitespace"));
         }
 
-        if (fieldName.equals(STRING_FIELD_NAME) && randomBoolean()) {
+        if (isTextField(fieldName) && randomBoolean()) {
             matchQuery.fuzziness(randomFuzziness(fieldName));
         }
 
@@ -179,6 +181,12 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
         if (query instanceof ExtendedCommonTermsQuery) {
             assertTrue(queryBuilder.cutoffFrequency() != null);
             ExtendedCommonTermsQuery ectq = (ExtendedCommonTermsQuery) query;
+            List<Term> terms = ectq.getTerms();
+            if (!terms.isEmpty()) {
+                Term term = terms.iterator().next();
+                String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+                assertThat(term.field(), equalTo(expectedFieldName));
+            }
             assertEquals(queryBuilder.cutoffFrequency(), ectq.getMaxTermFrequency(), Float.MIN_VALUE);
         }
 
@@ -195,6 +203,9 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
                 termLcMatcher = either(termLcMatcher).or(equalTo(originalTermLc.substring(0, 1)));
             }
             assertThat(actualTermLc, termLcMatcher);
+
+            String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+            assertThat(expectedFieldName, equalTo(fuzzyQuery.getTerm().field()));
             assertThat(queryBuilder.prefixLength(), equalTo(fuzzyQuery.getPrefixLength()));
             assertThat(queryBuilder.fuzzyTranspositions(), equalTo(fuzzyQuery.getTranspositions()));
         }
diff --git a/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index de044d58793..b54ce571453 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -91,7 +91,7 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase<MoreLik
     }
 
     private static String[] randomStringFields() {
-        String[] mappedStringFields = new String[]{STRING_FIELD_NAME, STRING_FIELD_NAME_2};
+        String[] mappedStringFields = new String[]{STRING_FIELD_NAME, STRING_FIELD_NAME_2, STRING_ALIAS_FIELD_NAME};
         String[] unmappedStringFields = generateRandomStringArray(2, 5, false, false);
         return Stream.concat(Arrays.stream(mappedStringFields), Arrays.stream(unmappedStringFields)).toArray(String[]::new);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
index b6d4816b01f..9eed4d9f1c1 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
@@ -230,9 +230,10 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
         assertThat(dQuery.getTieBreakerMultiplier(), equalTo(1.0f));
-        assertThat(dQuery.getDisjuncts().size(), equalTo(2));
-        assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
-        assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
+        assertThat(dQuery.getDisjuncts().size(), equalTo(3));
+        assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
+        assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
+        assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 2).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
     }
 
     public void testToQueryFieldMissing() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index 11cf6399745..c78c83c154f 100644
--- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -60,7 +60,9 @@ public class PrefixQueryBuilderTests extends AbstractQueryTestCase<PrefixQueryBu
     }
 
     private static PrefixQueryBuilder randomPrefixQuery() {
-        String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
+        String fieldName = randomFrom(STRING_FIELD_NAME,
+            STRING_ALIAS_FIELD_NAME,
+            randomAlphaOfLengthBetween(1, 10));
         String value = randomAlphaOfLengthBetween(1, 10);
         return new PrefixQueryBuilder(fieldName, value);
     }
@@ -69,7 +71,9 @@ public class PrefixQueryBuilderTests extends AbstractQueryTestCase<PrefixQueryBu
     protected void doAssertLuceneQuery(PrefixQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(PrefixQuery.class));
         PrefixQuery prefixQuery = (PrefixQuery) query;
-        assertThat(prefixQuery.getPrefix().field(), equalTo(queryBuilder.fieldName()));
+
+        String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+        assertThat(prefixQuery.getPrefix().field(), equalTo(expectedFieldName));
         assertThat(prefixQuery.getPrefix().text(), equalTo(queryBuilder.value()));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index 1c8bc48fb7e..4b9e0f5a66e 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -90,12 +90,16 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         }
         QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder(query);
         if (randomBoolean()) {
-            queryStringQueryBuilder.defaultField(randomBoolean() ?
-                STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10));
+            String defaultFieldName = randomFrom(STRING_FIELD_NAME,
+                STRING_ALIAS_FIELD_NAME,
+                randomAlphaOfLengthBetween(1, 10));
+            queryStringQueryBuilder.defaultField(defaultFieldName);
         } else {
             int numFields = randomIntBetween(1, 5);
             for (int i = 0; i < numFields; i++) {
-                String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
+                String fieldName = randomFrom(STRING_FIELD_NAME,
+                    STRING_ALIAS_FIELD_NAME,
+                    randomAlphaOfLengthBetween(1, 10));
                 if (randomBoolean()) {
                     queryStringQueryBuilder.field(fieldName);
                 } else {
@@ -508,10 +512,12 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         Query query = queryStringQuery("test").field("mapped_str*").toQuery(createShardContext());
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
-        assertThat(dQuery.getDisjuncts().size(), equalTo(2));
+        assertThat(dQuery.getDisjuncts().size(), equalTo(3));
         assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(),
-            equalTo(new Term(STRING_FIELD_NAME_2, "test")));
+            equalTo(new Term(STRING_FIELD_NAME, "test")));
         assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(),
+            equalTo(new Term(STRING_FIELD_NAME_2, "test")));
+        assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 2).getTerm(),
             equalTo(new Term(STRING_FIELD_NAME, "test")));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java b/server/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
index 9b064485592..ecd767b9d65 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
@@ -22,10 +22,12 @@ package org.elasticsearch.index.query;
 import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 
-import org.elasticsearch.test.AbstractQueryTestCase;
-
 import java.util.Random;
 
+import static org.elasticsearch.test.AbstractBuilderTestCase.STRING_ALIAS_FIELD_NAME;
+import static org.elasticsearch.test.AbstractBuilderTestCase.STRING_FIELD_NAME;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+
 /**
  * Utility class for creating random QueryBuilders.
  * So far only leaf queries like {@link MatchAllQueryBuilder}, {@link TermQueryBuilder} or
@@ -62,9 +64,10 @@ public class RandomQueryBuilder {
         // for now, only use String Rangequeries for MultiTerm test, numeric and date makes little sense
         // see issue #12123 for discussion
         MultiTermQueryBuilder multiTermQueryBuilder;
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME);
         switch(RandomNumbers.randomIntBetween(r, 0, 3)) {
             case 0:
-                RangeQueryBuilder stringRangeQuery = new RangeQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME);
+                RangeQueryBuilder stringRangeQuery = new RangeQueryBuilder(fieldName);
                 stringRangeQuery.from("a" + RandomStrings.randomAsciiOfLengthBetween(r, 1, 10));
                 stringRangeQuery.to("z" + RandomStrings.randomAsciiOfLengthBetween(r, 1, 10));
                 multiTermQueryBuilder = stringRangeQuery;
@@ -76,7 +79,7 @@ public class RandomQueryBuilder {
                 multiTermQueryBuilder = new WildcardQueryBuilderTests().createTestQueryBuilder();
                 break;
             case 3:
-                multiTermQueryBuilder = new FuzzyQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME,
+                multiTermQueryBuilder = new FuzzyQueryBuilder(fieldName,
                         RandomStrings.randomAsciiOfLengthBetween(r, 1, 10));
                 break;
             default:
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index 3668c7dec17..ba02ddaf05c 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -65,13 +65,15 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
         switch (randomIntBetween(0, 2)) {
             case 0:
                 // use mapped integer field for numeric range queries
-                query = new RangeQueryBuilder(randomBoolean() ? INT_FIELD_NAME : INT_RANGE_FIELD_NAME);
+                query = new RangeQueryBuilder(randomFrom(
+                    INT_FIELD_NAME, INT_RANGE_FIELD_NAME, INT_ALIAS_FIELD_NAME));
                 query.from(randomIntBetween(1, 100));
                 query.to(randomIntBetween(101, 200));
                 break;
             case 1:
                 // use mapped date field, using date string representation
-                query = new RangeQueryBuilder(randomBoolean() ? DATE_FIELD_NAME : DATE_RANGE_FIELD_NAME);
+                query = new RangeQueryBuilder(randomFrom(
+                    DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, DATE_ALIAS_FIELD_NAME));
                 query.from(new DateTime(System.currentTimeMillis() - randomIntBetween(0, 1000000), DateTimeZone.UTC).toString());
                 query.to(new DateTime(System.currentTimeMillis() + randomIntBetween(0, 1000000), DateTimeZone.UTC).toString());
                 // Create timestamp option only then we have a date mapper,
@@ -87,7 +89,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
                 break;
             case 2:
             default:
-                query = new RangeQueryBuilder(STRING_FIELD_NAME);
+                query = new RangeQueryBuilder(randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME));
                 query.from("a" + randomAlphaOfLengthBetween(1, 10));
                 query.to("z" + randomAlphaOfLengthBetween(1, 10));
                 break;
@@ -128,17 +130,18 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
 
     @Override
     protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
         if (queryBuilder.from() == null && queryBuilder.to() == null) {
             final Query expectedQuery;
             if (getCurrentTypes().length > 0) {
                 if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
                         && context.mapperService().fullName(queryBuilder.fieldName()).hasDocValues()) {
-                    expectedQuery = new ConstantScoreQuery(new DocValuesFieldExistsQuery(queryBuilder.fieldName()));
+                    expectedQuery = new ConstantScoreQuery(new DocValuesFieldExistsQuery(expectedFieldName));
                 } else if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
                         && context.mapperService().fullName(queryBuilder.fieldName()).omitNorms() == false) {
-                    expectedQuery = new ConstantScoreQuery(new NormsFieldExistsQuery(queryBuilder.fieldName()));
+                    expectedQuery = new ConstantScoreQuery(new NormsFieldExistsQuery(expectedFieldName));
                 } else {
-                    expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, queryBuilder.fieldName())));
+                    expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, expectedFieldName)));
                 }
             } else {
                 expectedQuery = new MatchNoDocsQuery("no mappings yet");
@@ -146,18 +149,18 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
             assertThat(query, equalTo(expectedQuery));
 
         } else if (getCurrentTypes().length == 0 ||
-            (queryBuilder.fieldName().equals(DATE_FIELD_NAME) == false
-                && queryBuilder.fieldName().equals(INT_FIELD_NAME) == false
-                && queryBuilder.fieldName().equals(DATE_RANGE_FIELD_NAME) == false
-                && queryBuilder.fieldName().equals(INT_RANGE_FIELD_NAME) == false)) {
+            (expectedFieldName.equals(DATE_FIELD_NAME) == false
+                && expectedFieldName.equals(INT_FIELD_NAME) == false
+                && expectedFieldName.equals(DATE_RANGE_FIELD_NAME) == false
+                && expectedFieldName.equals(INT_RANGE_FIELD_NAME) == false)) {
             assertThat(query, instanceOf(TermRangeQuery.class));
             TermRangeQuery termRangeQuery = (TermRangeQuery) query;
-            assertThat(termRangeQuery.getField(), equalTo(queryBuilder.fieldName()));
+            assertThat(termRangeQuery.getField(), equalTo(expectedFieldName));
             assertThat(termRangeQuery.getLowerTerm(), equalTo(BytesRefs.toBytesRef(queryBuilder.from())));
             assertThat(termRangeQuery.getUpperTerm(), equalTo(BytesRefs.toBytesRef(queryBuilder.to())));
             assertThat(termRangeQuery.includesLower(), equalTo(queryBuilder.includeLower()));
             assertThat(termRangeQuery.includesUpper(), equalTo(queryBuilder.includeUpper()));
-        } else if (queryBuilder.fieldName().equals(DATE_FIELD_NAME)) {
+        } else if (expectedFieldName.equals(DATE_FIELD_NAME)) {
             assertThat(query, instanceOf(IndexOrDocValuesQuery.class));
             query = ((IndexOrDocValuesQuery) query).getIndexQuery();
             assertThat(query, instanceOf(PointRangeQuery.class));
@@ -202,7 +205,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
                 }
             }
             assertEquals(LongPoint.newRangeQuery(DATE_FIELD_NAME, minLong, maxLong), query);
-        } else if (queryBuilder.fieldName().equals(INT_FIELD_NAME)) {
+        } else if (expectedFieldName.equals(INT_FIELD_NAME)) {
             assertThat(query, instanceOf(IndexOrDocValuesQuery.class));
             query = ((IndexOrDocValuesQuery) query).getIndexQuery();
             assertThat(query, instanceOf(PointRangeQuery.class));
@@ -225,7 +228,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
                     maxInt--;
                 }
             }
-        } else if (queryBuilder.fieldName().equals(DATE_RANGE_FIELD_NAME) || queryBuilder.fieldName().equals(INT_RANGE_FIELD_NAME)) {
+        } else if (expectedFieldName.equals(DATE_RANGE_FIELD_NAME) || expectedFieldName.equals(INT_RANGE_FIELD_NAME)) {
             // todo can't check RangeFieldQuery because its currently package private (this will change)
         } else {
             throw new UnsupportedOperationException();
diff --git a/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
index efbb84c3239..1110388090a 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
@@ -71,7 +71,7 @@ public class RegexpQueryBuilderTests extends AbstractQueryTestCase<RegexpQueryBu
 
     private static RegexpQueryBuilder randomRegexpQuery() {
         // mapped or unmapped fields
-        String fieldName = randomBoolean() ? STRING_FIELD_NAME : randomAlphaOfLengthBetween(1, 10);
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, randomAlphaOfLengthBetween(1, 10));
         String value = randomAlphaOfLengthBetween(1, 10);
         return new RegexpQueryBuilder(fieldName, value);
     }
@@ -80,7 +80,9 @@ public class RegexpQueryBuilderTests extends AbstractQueryTestCase<RegexpQueryBu
     protected void doAssertLuceneQuery(RegexpQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(RegexpQuery.class));
         RegexpQuery regexpQuery = (RegexpQuery) query;
-        assertThat(regexpQuery.getField(), equalTo(queryBuilder.fieldName()));
+
+        String expectedFieldName = expectedFieldName( queryBuilder.fieldName());
+        assertThat(regexpQuery.getField(), equalTo(expectedFieldName));
     }
 
     public void testIllegalArguments() {
diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index 6be8d26a9d5..3c508ddc8be 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -98,7 +98,8 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
         Map<String, Float> fields = new HashMap<>();
         for (int i = 0; i < fieldCount; i++) {
             if (randomBoolean()) {
-                fields.put(STRING_FIELD_NAME, AbstractQueryBuilder.DEFAULT_BOOST);
+                String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME);
+                fields.put(fieldName, AbstractQueryBuilder.DEFAULT_BOOST);
             } else {
                 fields.put(STRING_FIELD_NAME_2, 2.0f / randomIntBetween(1, 20));
             }
diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
index c93df5b7519..7c459737c77 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -63,7 +63,11 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase<SpanMu
                 .field("type", "text")
                 .startObject("index_prefixes").endObject()
             .endObject()
-            .endObject().endObject().endObject();
+            .startObject("prefix_field_alias")
+                .field("type", "alias")
+                .field("path", "prefix_field")
+            .endObject()
+        .endObject().endObject().endObject();
 
         mapperService.merge("_doc",
             new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE);
@@ -168,16 +172,16 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase<SpanMu
     }
 
     public void testToQueryInnerSpanMultiTerm() throws IOException {
-
         Query query = new SpanOrQueryBuilder(createTestQueryBuilder()).toQuery(createShardContext());
         //verify that the result is still a span query, despite the boost that might get set (SpanBoostQuery rather than BoostQuery)
         assertThat(query, instanceOf(SpanQuery.class));
     }
 
     public void testToQueryInnerTermQuery() throws IOException {
+        String fieldName = randomFrom("prefix_field", "prefix_field_alias");
         final QueryShardContext context = createShardContext();
         if (context.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_4_0)) {
-            Query query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder("prefix_field", "foo"))
+            Query query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder(fieldName, "foo"))
                 .toQuery(context);
             assertThat(query, instanceOf(FieldMaskingSpanQuery.class));
             FieldMaskingSpanQuery fieldSpanQuery = (FieldMaskingSpanQuery) query;
@@ -186,7 +190,7 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase<SpanMu
             SpanTermQuery spanTermQuery = (SpanTermQuery) fieldSpanQuery.getMaskedQuery();
             assertThat(spanTermQuery.getTerm().text(), equalTo("foo"));
 
-            query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder("prefix_field", "foo"))
+            query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder(fieldName, "foo"))
                 .boost(2.0f)
                 .toQuery(context);
             assertThat(query, instanceOf(SpanBoostQuery.class));
@@ -199,7 +203,7 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase<SpanMu
             spanTermQuery = (SpanTermQuery) fieldSpanQuery.getMaskedQuery();
             assertThat(spanTermQuery.getTerm().text(), equalTo("foo"));
         } else {
-            Query query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder("prefix_field", "foo"))
+            Query query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder(fieldName, "foo"))
                 .toQuery(context);
             assertThat(query, instanceOf(SpanMultiTermQueryWrapper.class));
             SpanMultiTermQueryWrapper wrapper = (SpanMultiTermQueryWrapper) query;
@@ -208,7 +212,7 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase<SpanMu
             assertThat(prefixQuery.getField(), equalTo("prefix_field"));
             assertThat(prefixQuery.getPrefix().text(), equalTo("foo"));
 
-            query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder("prefix_field", "foo"))
+            query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder(fieldName, "foo"))
                 .boost(2.0f)
                 .toQuery(context);
             assertThat(query, instanceOf(SpanBoostQuery.class));
diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java
index d21a18c0839..34dc08d29fb 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java
@@ -38,12 +38,11 @@ public class SpanTermQueryBuilderTests extends AbstractTermQueryTestCase<SpanTer
 
     @Override
     protected SpanTermQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = null;
-        Object value;
+        String fieldName = randomFrom(STRING_FIELD_NAME,
+            STRING_ALIAS_FIELD_NAME,
+            randomAlphaOfLengthBetween(1, 10));
 
-        if (randomBoolean()) {
-            fieldName = STRING_FIELD_NAME;
-        }
+        Object value;
         if (frequently()) {
             value = randomAlphaOfLengthBetween(1, 10);
         } else {
@@ -51,10 +50,6 @@ public class SpanTermQueryBuilderTests extends AbstractTermQueryTestCase<SpanTer
             JsonStringEncoder encoder = JsonStringEncoder.getInstance();
             value = new String(encoder.quoteAsString(randomUnicodeOfLength(10)));
         }
-
-        if (fieldName == null) {
-            fieldName = randomAlphaOfLengthBetween(1, 10);
-        }
         return createQueryBuilder(fieldName, value);
     }
 
@@ -67,7 +62,10 @@ public class SpanTermQueryBuilderTests extends AbstractTermQueryTestCase<SpanTer
     protected void doAssertLuceneQuery(SpanTermQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(SpanTermQuery.class));
         SpanTermQuery spanTermQuery = (SpanTermQuery) query;
-        assertThat(spanTermQuery.getTerm().field(), equalTo(queryBuilder.fieldName()));
+
+        String expectedFieldName = expectedFieldName(queryBuilder.fieldName);
+        assertThat(spanTermQuery.getTerm().field(), equalTo(expectedFieldName));
+
         MappedFieldType mapper = context.getQueryShardContext().fieldMapper(queryBuilder.fieldName());
         if (mapper != null) {
             Term term = ((TermQuery) mapper.termQuery(queryBuilder.value(), null)).getTerm();
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
index 45cbd43864f..b6cc150717a 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
@@ -50,7 +50,7 @@ public class TermQueryBuilderTests extends AbstractTermQueryTestCase<TermQueryBu
                 break;
             case 1:
                 if (randomBoolean()) {
-                    fieldName = STRING_FIELD_NAME;
+                    fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME);
                 }
                 if (frequently()) {
                     value = randomAlphaOfLengthBetween(1, 10);
@@ -96,7 +96,10 @@ public class TermQueryBuilderTests extends AbstractTermQueryTestCase<TermQueryBu
         MappedFieldType mapper = context.getQueryShardContext().fieldMapper(queryBuilder.fieldName());
         if (query instanceof TermQuery) {
             TermQuery termQuery = (TermQuery) query;
-            assertThat(termQuery.getTerm().field(), equalTo(queryBuilder.fieldName()));
+
+            String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+            assertThat(termQuery.getTerm().field(), equalTo(expectedFieldName));
+
             if (mapper != null) {
                 Term term = ((TermQuery) mapper.termQuery(queryBuilder.value(), null)).getTerm();
                 assertThat(termQuery.getTerm(), equalTo(term));
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index baacd13809b..963a08ce753 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -77,9 +77,13 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
         // terms query or lookup query
         if (randomBoolean()) {
             // make between 0 and 5 different values of the same type
-            String fieldName;
-            fieldName = randomValueOtherThanMany(choice -> choice.equals(GEO_POINT_FIELD_NAME) || choice.equals(GEO_SHAPE_FIELD_NAME)
-                    || choice.equals(INT_RANGE_FIELD_NAME) || choice.equals(DATE_RANGE_FIELD_NAME), () -> getRandomFieldName());
+            String fieldName = randomValueOtherThanMany(choice ->
+                    choice.equals(GEO_POINT_FIELD_NAME) ||
+                    choice.equals(GEO_POINT_ALIAS_FIELD_NAME) ||
+                    choice.equals(GEO_SHAPE_FIELD_NAME) ||
+                    choice.equals(INT_RANGE_FIELD_NAME) ||
+                    choice.equals(DATE_RANGE_FIELD_NAME),
+                () -> getRandomFieldName());
             Object[] values = new Object[randomInt(5)];
             for (int i = 0; i < values.length; i++) {
                 values[i] = getRandomValueForFieldName(fieldName);
@@ -129,7 +133,8 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
                 terms = queryBuilder.values();
             }
 
-            TermInSetQuery expected = new TermInSetQuery(queryBuilder.fieldName(),
+            String fieldName = expectedFieldName(queryBuilder.fieldName());
+            TermInSetQuery expected = new TermInSetQuery(fieldName,
                     terms.stream().filter(Objects::nonNull).map(Object::toString).map(BytesRef::new).collect(Collectors.toList()));
             assertEquals(expected, query);
         }
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
index b3cd1a361ec..cfc423d918a 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
@@ -28,12 +28,14 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CoveringQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
@@ -83,10 +85,9 @@ public class TermsSetQueryBuilderTests extends AbstractQueryTestCase<TermsSetQue
 
     @Override
     protected TermsSetQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName;
-        do {
-            fieldName = randomFrom(MAPPED_FIELD_NAMES);
-        } while (fieldName.equals(GEO_POINT_FIELD_NAME) || fieldName.equals(GEO_SHAPE_FIELD_NAME));
+        String fieldName = randomValueOtherThanMany(
+            value -> value.equals(GEO_POINT_FIELD_NAME) || value.equals(GEO_SHAPE_FIELD_NAME),
+            () -> randomFrom(MAPPED_FIELD_NAMES));
         List<?> randomTerms = randomValues(fieldName);
         TermsSetQueryBuilder queryBuilder = new TermsSetQueryBuilder(STRING_FIELD_NAME, randomTerms);
         if (randomBoolean()) {
@@ -261,6 +262,22 @@ public class TermsSetQueryBuilderTests extends AbstractQueryTestCase<TermsSetQue
         }
     }
 
+    public void testFieldAlias() {
+        List<String> randomTerms = Arrays.asList(generateRandomStringArray(5, 10, false, false));
+        TermsSetQueryBuilder queryBuilder = new TermsSetQueryBuilder(STRING_ALIAS_FIELD_NAME, randomTerms)
+            .setMinimumShouldMatchField("m_s_m");
+
+        QueryShardContext context = createShardContext();
+        List<Query> termQueries = queryBuilder.createTermQueries(context);
+        assertEquals(randomTerms.size(), termQueries.size());
+
+        String expectedFieldName = expectedFieldName(queryBuilder.getFieldName());
+        for (int i = 0; i < randomTerms.size(); i++) {
+            Term term = new Term(expectedFieldName, randomTerms.get(i));
+            assertThat(termQueries.get(i), equalTo(new TermQuery(term)));
+        }
+    }
+
     private static List<?> randomValues(final String fieldName) {
         final int numValues = randomIntBetween(0, 10);
         final List<Object> values = new ArrayList<>(numValues);
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 7da423de25b..48f43eefeb3 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -60,21 +60,22 @@ public class WildcardQueryBuilderTests extends AbstractQueryTestCase<WildcardQue
     }
 
     private static WildcardQueryBuilder randomWildcardQuery() {
-        // mapped or unmapped field
+        String fieldName = randomFrom(STRING_FIELD_NAME,
+            STRING_ALIAS_FIELD_NAME,
+            randomAlphaOfLengthBetween(1, 10));
         String text = randomAlphaOfLengthBetween(1, 10);
-        if (randomBoolean()) {
-            return new WildcardQueryBuilder(STRING_FIELD_NAME, text);
-        } else {
-            return new WildcardQueryBuilder(randomAlphaOfLengthBetween(1, 10), text);
-        }
+
+        return new WildcardQueryBuilder(fieldName, text);
     }
 
     @Override
     protected void doAssertLuceneQuery(WildcardQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
         assertThat(query, instanceOf(WildcardQuery.class));
         WildcardQuery wildcardQuery = (WildcardQuery) query;
-        assertThat(wildcardQuery.getField(), equalTo(queryBuilder.fieldName()));
-        assertThat(wildcardQuery.getTerm().field(), equalTo(queryBuilder.fieldName()));
+
+        String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
+        assertThat(wildcardQuery.getField(), equalTo(expectedFieldName));
+        assertThat(wildcardQuery.getTerm().field(), equalTo(expectedFieldName));
         assertThat(wildcardQuery.getTerm().text(), equalTo(queryBuilder.value()));
     }
 
@@ -138,19 +139,19 @@ public class WildcardQueryBuilderTests extends AbstractQueryTestCase<WildcardQue
             assertEquals(expected, query);
         }
     }
-    
+
     public void testIndexWildcard() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
 
         QueryShardContext context = createShardContext();
         String index = context.getFullyQualifiedIndexName();
-        
+
         Query query = new WildcardQueryBuilder("_index", index).doToQuery(context);
         assertThat(query instanceof MatchAllDocsQuery, equalTo(true));
-        
+
         query = new WildcardQueryBuilder("_index", index + "*").doToQuery(context);
         assertThat(query instanceof MatchAllDocsQuery, equalTo(true));
-        
+
         query = new WildcardQueryBuilder("_index", "index_" + index + "*").doToQuery(context);
         assertThat(query instanceof MatchNoDocsQuery, equalTo(true));
     }
diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
index 3de02f68318..6102a1b55f1 100644
--- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
+++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
@@ -38,11 +38,12 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -86,22 +87,21 @@ public class SimilarityTests extends ESSingleNodeTestCase {
     }
 
     public void testResolveSimilaritiesFromMapping_classic() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
             .startObject("field1").field("type", "text").field("similarity", "my_similarity").endObject()
             .endObject()
-            .endObject().endObject());
+            .endObject().endObject();
 
         Settings indexSettings = Settings.builder()
             .put("index.similarity.my_similarity.type", "classic")
             .put("index.similarity.my_similarity.discount_overlaps", false)
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) // otherwise classic is forbidden
             .build();
-        IndexService indexService = createIndex("foo", indexSettings);
-        DocumentMapper documentMapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity().get(), instanceOf(ClassicSimilarity.class));
+        MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
+        assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(ClassicSimilarity.class));
 
-        ClassicSimilarity similarity = (ClassicSimilarity) documentMapper.mappers().getMapper("field1").fieldType().similarity().get();
+        ClassicSimilarity similarity = (ClassicSimilarity) mapperService.fullName("field1").similarity().get();
         assertThat(similarity.getDiscountOverlaps(), equalTo(false));
     }
 
@@ -117,11 +117,11 @@ public class SimilarityTests extends ESSingleNodeTestCase {
     }
 
     public void testResolveSimilaritiesFromMapping_bm25() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
             .startObject("field1").field("type", "text").field("similarity", "my_similarity").endObject()
             .endObject()
-            .endObject().endObject());
+            .endObject().endObject();
 
         Settings indexSettings = Settings.builder()
             .put("index.similarity.my_similarity.type", "BM25")
@@ -129,37 +129,32 @@ public class SimilarityTests extends ESSingleNodeTestCase {
             .put("index.similarity.my_similarity.b", 0.5f)
             .put("index.similarity.my_similarity.discount_overlaps", false)
             .build();
-        IndexService indexService = createIndex("foo", indexSettings);
-        DocumentMapper documentMapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity().get(), instanceOf(BM25Similarity.class));
+        MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
+        assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(BM25Similarity.class));
 
-        BM25Similarity similarity = (BM25Similarity) documentMapper.mappers().getMapper("field1").fieldType().similarity().get();
+        BM25Similarity similarity = (BM25Similarity) mapperService.fullName("field1").similarity().get();
         assertThat(similarity.getK1(), equalTo(2.0f));
         assertThat(similarity.getB(), equalTo(0.5f));
         assertThat(similarity.getDiscountOverlaps(), equalTo(false));
     }
 
     public void testResolveSimilaritiesFromMapping_boolean() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
             .startObject("field1").field("type", "text").field("similarity", "boolean").endObject()
             .endObject()
-            .endObject().endObject());
+            .endObject().endObject();
 
-        IndexService indexService = createIndex("foo", Settings.EMPTY);
-        DocumentMapper documentMapper = indexService.mapperService()
-            .documentMapperParser()
-            .parse("type", new CompressedXContent(mapping));
-        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity().get(),
-            instanceOf(BooleanSimilarity.class));
+        MapperService mapperService = createIndex("foo", Settings.EMPTY, "type", mapping).mapperService();
+        assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(BooleanSimilarity.class));
     }
 
     public void testResolveSimilaritiesFromMapping_DFR() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
             .startObject("field1").field("type", "text").field("similarity", "my_similarity").endObject()
             .endObject()
-            .endObject().endObject());
+            .endObject().endObject();
 
         Settings indexSettings = Settings.builder()
             .put("index.similarity.my_similarity.type", "DFR")
@@ -168,11 +163,10 @@ public class SimilarityTests extends ESSingleNodeTestCase {
             .put("index.similarity.my_similarity.normalization", "h2")
             .put("index.similarity.my_similarity.normalization.h2.c", 3f)
             .build();
-        IndexService indexService = createIndex("foo", indexSettings);
-        DocumentMapper documentMapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity().get(), instanceOf(DFRSimilarity.class));
+        MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
+        assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(DFRSimilarity.class));
 
-        DFRSimilarity similarity = (DFRSimilarity) documentMapper.mappers().getMapper("field1").fieldType().similarity().get();
+        DFRSimilarity similarity = (DFRSimilarity) mapperService.fullName("field1").similarity().get();
         assertThat(similarity.getBasicModel(), instanceOf(BasicModelG.class));
         assertThat(similarity.getAfterEffect(), instanceOf(AfterEffectL.class));
         assertThat(similarity.getNormalization(), instanceOf(NormalizationH2.class));
@@ -180,11 +174,11 @@ public class SimilarityTests extends ESSingleNodeTestCase {
     }
 
     public void testResolveSimilaritiesFromMapping_IB() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
             .startObject("field1").field("type", "text").field("similarity", "my_similarity").endObject()
             .endObject()
-            .endObject().endObject());
+            .endObject().endObject();
 
         Settings indexSettings = Settings.builder()
             .put("index.similarity.my_similarity.type", "IB")
@@ -193,11 +187,10 @@ public class SimilarityTests extends ESSingleNodeTestCase {
             .put("index.similarity.my_similarity.normalization", "h2")
             .put("index.similarity.my_similarity.normalization.h2.c", 3f)
             .build();
-        IndexService indexService = createIndex("foo", indexSettings);
-        DocumentMapper documentMapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity().get(), instanceOf(IBSimilarity.class));
+        MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
+        assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(IBSimilarity.class));
 
-        IBSimilarity similarity = (IBSimilarity) documentMapper.mappers().getMapper("field1").fieldType().similarity().get();
+        IBSimilarity similarity = (IBSimilarity) mapperService.fullName("field1").similarity().get();
         assertThat(similarity.getDistribution(), instanceOf(DistributionSPL.class));
         assertThat(similarity.getLambda(), instanceOf(LambdaTTF.class));
         assertThat(similarity.getNormalization(), instanceOf(NormalizationH2.class));
@@ -205,59 +198,58 @@ public class SimilarityTests extends ESSingleNodeTestCase {
     }
 
     public void testResolveSimilaritiesFromMapping_DFI() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
             .startObject("field1").field("type", "text").field("similarity", "my_similarity").endObject()
             .endObject()
-            .endObject().endObject());
+            .endObject().endObject();
 
         Settings indexSettings = Settings.builder()
             .put("index.similarity.my_similarity.type", "DFI")
             .put("index.similarity.my_similarity.independence_measure", "chisquared")
             .build();
-        IndexService indexService = createIndex("foo", indexSettings);
-        DocumentMapper documentMapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        MappedFieldType fieldType = documentMapper.mappers().getMapper("field1").fieldType();
+        MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
+        MappedFieldType fieldType = mapperService.fullName("field1");
+
         assertThat(fieldType.similarity().get(), instanceOf(DFISimilarity.class));
         DFISimilarity similarity = (DFISimilarity) fieldType.similarity().get();
         assertThat(similarity.getIndependence(), instanceOf(IndependenceChiSquared.class));
     }
 
     public void testResolveSimilaritiesFromMapping_LMDirichlet() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
             .startObject("field1").field("type", "text").field("similarity", "my_similarity").endObject()
             .endObject()
-            .endObject().endObject());
+            .endObject().endObject();
 
         Settings indexSettings = Settings.builder()
             .put("index.similarity.my_similarity.type", "LMDirichlet")
             .put("index.similarity.my_similarity.mu", 3000f)
             .build();
-        IndexService indexService = createIndex("foo", indexSettings);
-        DocumentMapper documentMapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity().get(), instanceOf(LMDirichletSimilarity.class));
 
-        LMDirichletSimilarity similarity = (LMDirichletSimilarity) documentMapper.mappers().getMapper("field1").fieldType().similarity().get();
+        MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
+        assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(LMDirichletSimilarity.class));
+
+        LMDirichletSimilarity similarity = (LMDirichletSimilarity) mapperService.fullName("field1").similarity().get();
         assertThat(similarity.getMu(), equalTo(3000f));
     }
 
     public void testResolveSimilaritiesFromMapping_LMJelinekMercer() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties")
             .startObject("field1").field("type", "text").field("similarity", "my_similarity").endObject()
             .endObject()
-            .endObject().endObject());
+            .endObject().endObject();
 
         Settings indexSettings = Settings.builder()
             .put("index.similarity.my_similarity.type", "LMJelinekMercer")
             .put("index.similarity.my_similarity.lambda", 0.7f)
             .build();
-        IndexService indexService = createIndex("foo", indexSettings);
-        DocumentMapper documentMapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        assertThat(documentMapper.mappers().getMapper("field1").fieldType().similarity().get(), instanceOf(LMJelinekMercerSimilarity.class));
+        MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService();
+        assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(LMJelinekMercerSimilarity.class));
 
-        LMJelinekMercerSimilarity similarity = (LMJelinekMercerSimilarity) documentMapper.mappers().getMapper("field1").fieldType().similarity().get();
+        LMJelinekMercerSimilarity similarity = (LMJelinekMercerSimilarity) mapperService.fullName("field1").similarity().get();
         assertThat(similarity.getLambda(), equalTo(0.7f));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java b/server/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
index 2ba943ba0dc..788777ade7d 100644
--- a/server/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.indices.mapping;
 
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -68,10 +69,26 @@ public class SimpleGetFieldMappingsIT extends ESIntegTestCase {
     }
 
     private XContentBuilder getMappingForType(String type) throws IOException {
-        return jsonBuilder().startObject().startObject(type).startObject("properties")
-                .startObject("field1").field("type", "text").endObject()
-                .startObject("obj").startObject("properties").startObject("subfield").field("type", "keyword").endObject().endObject().endObject()
-                .endObject().endObject().endObject();
+        return jsonBuilder().startObject()
+            .startObject(type)
+                .startObject("properties")
+                    .startObject("field1")
+                        .field("type", "text")
+                    .endObject()
+                   .startObject("alias")
+                        .field("type", "alias")
+                        .field("path", "field1")
+                    .endObject()
+                    .startObject("obj")
+                        .startObject("properties")
+                            .startObject("subfield")
+                                .field("type", "keyword")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject();
     }
 
     public void testGetFieldMappings() throws Exception {
@@ -138,8 +155,23 @@ public class SimpleGetFieldMappingsIT extends ESIntegTestCase {
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "field1").sourceAsMap().get("field1"), hasEntry("index", Boolean.TRUE));
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "field1").sourceAsMap().get("field1"), hasEntry("type", (Object) "text"));
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "obj.subfield").sourceAsMap().get("subfield"), hasEntry("type", (Object) "keyword"));
+    }
 
+    @SuppressWarnings("unchecked")
+    public void testGetFieldMappingsWithFieldAlias() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("type", getMappingForType("type")));
 
+        GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings()
+            .setFields("alias", "field1").get();
+
+        FieldMappingMetaData aliasMapping = response.fieldMappings("test", "type", "alias");
+        assertThat(aliasMapping.fullName(), equalTo("alias"));
+        assertThat(aliasMapping.sourceAsMap(), hasKey("alias"));
+        assertThat((Map<String, Object>) aliasMapping.sourceAsMap().get("alias"), hasEntry("type", "alias"));
+
+        FieldMappingMetaData field1Mapping = response.fieldMappings("test", "type", "field1");
+        assertThat(field1Mapping.fullName(), equalTo("field1"));
+        assertThat(field1Mapping.sourceAsMap(), hasKey("field1"));
     }
 
     //fix #6552
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
index c2a2405098d..edc29b0d2c5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
@@ -116,6 +116,21 @@ public class RangeIT extends ESIntegTestCase {
                     .field(SINGLE_VALUED_FIELD_NAME, i * 2 - 1)
                     .endObject()));
         }
+
+        // Create two indices and add the field 'route_length_miles' as an alias in
+        // one, and a concrete field in the other.
+        prepareCreate("old_index")
+            .addMapping("_doc", "distance", "type=double", "route_length_miles", "type=alias,path=distance")
+            .get();
+        prepareCreate("new_index")
+            .addMapping("_doc", "route_length_miles", "type=double")
+            .get();
+
+        builders.add(client().prepareIndex("old_index", "_doc").setSource("distance", 42.0));
+        builders.add(client().prepareIndex("old_index", "_doc").setSource("distance", 50.5));
+        builders.add(client().prepareIndex("new_index", "_doc").setSource("route_length_miles", 100.2));
+        builders.add(client().prepareIndex("new_index", "_doc").setSource(Collections.emptyMap()));
+
         indexRandom(true, builders);
         ensureSearchable();
     }
@@ -972,4 +987,72 @@ public class RangeIT extends ESIntegTestCase {
         assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
                 .getMissCount(), equalTo(1L));
     }
+
+    public void testFieldAlias() {
+        SearchResponse response = client().prepareSearch("old_index", "new_index")
+            .addAggregation(range("range")
+                .field("route_length_miles")
+                .addUnboundedTo(50.0)
+                .addRange(50.0, 150.0)
+                .addUnboundedFrom(150.0))
+            .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        Range range = response.getAggregations().get("range");
+        assertThat(range, notNullValue());
+        assertThat(range.getName(), equalTo("range"));
+        List<? extends Range.Bucket> buckets = range.getBuckets();
+        assertThat(buckets.size(), equalTo(3));
+
+        Range.Bucket bucket = buckets.get(0);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("*-50.0"));
+        assertThat(bucket.getDocCount(), equalTo(1L));
+
+        bucket = buckets.get(1);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("50.0-150.0"));
+        assertThat(bucket.getDocCount(), equalTo(2L));
+
+        bucket = buckets.get(2);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("150.0-*"));
+        assertThat(bucket.getDocCount(), equalTo(0L));
+    }
+
+
+    public void testFieldAliasWithMissingValue() {
+        SearchResponse response = client().prepareSearch("old_index", "new_index")
+            .addAggregation(range("range")
+                .field("route_length_miles")
+                .missing(0.0)
+                .addUnboundedTo(50.0)
+                .addRange(50.0, 150.0)
+                .addUnboundedFrom(150.0))
+            .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        Range range = response.getAggregations().get("range");
+        assertThat(range, notNullValue());
+        assertThat(range.getName(), equalTo("range"));
+        List<? extends Range.Bucket> buckets = range.getBuckets();
+        assertThat(buckets.size(), equalTo(3));
+
+        Range.Bucket bucket = buckets.get(0);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("*-50.0"));
+        assertThat(bucket.getDocCount(), equalTo(2L));
+
+        bucket = buckets.get(1);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("50.0-150.0"));
+        assertThat(bucket.getDocCount(), equalTo(2L));
+
+        bucket = buckets.get(2);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("150.0-*"));
+        assertThat(bucket.getDocCount(), equalTo(0L));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
index aaf366c7c7b..4a69f9d5379 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
@@ -66,6 +66,10 @@ public class ReverseNestedIT extends ESIntegTestCase {
                         "type",
                         jsonBuilder().startObject().startObject("properties")
                                 .startObject("field1").field("type", "keyword").endObject()
+                                .startObject("alias")
+                                    .field("type", "alias")
+                                    .field("path", "field1")
+                                .endObject()
                                 .startObject("nested1").field("type", "nested").startObject("properties")
                                     .startObject("field2").field("type", "keyword").endObject()
                                 .endObject().endObject()
@@ -649,4 +653,28 @@ public class ReverseNestedIT extends ESIntegTestCase {
             assertThat(barCount.getValue(), equalTo(2L));
         }
     }
+
+    public void testFieldAlias() {
+        SearchResponse response = client().prepareSearch("idx1")
+            .addAggregation(nested("nested1", "nested1")
+                .subAggregation(
+                    terms("field2").field("nested1.field2")
+                        .subAggregation(
+                            reverseNested("nested1_to_field1")
+                                .subAggregation(
+                                    terms("field1").field("alias")
+                                        .collectMode(randomFrom(SubAggCollectionMode.values())))))).get();
+
+        assertSearchResponse(response);
+
+        Nested nested = response.getAggregations().get("nested1");
+        Terms nestedTerms = nested.getAggregations().get("field2");
+        Terms.Bucket bucket = nestedTerms.getBuckets().iterator().next();
+
+        ReverseNested reverseNested = bucket.getAggregations().get("nested1_to_field1");
+        Terms reverseNestedTerms = reverseNested.getAggregations().get("field1");
+
+        assertThat(((InternalAggregation)reverseNested).getProperty("field1"), sameInstance(reverseNestedTerms));
+        assertThat(reverseNestedTerms.getBuckets().size(), equalTo(6));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
index 11a44a1d89b..4555809bad9 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
@@ -71,8 +71,14 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 import java.util.stream.DoubleStream;
 
+import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.nested;
+
 public class NestedAggregatorTests extends AggregatorTestCase {
 
     private static final String VALUE_FIELD_NAME = "number";
@@ -84,6 +90,15 @@ public class NestedAggregatorTests extends AggregatorTestCase {
 
     private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
 
+    /**
+     * For each provided field type, we also register an alias with name <field>-alias.
+     */
+    @Override
+    protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {
+        return Arrays.stream(fieldTypes).collect(Collectors.toMap(
+            ft -> ft.name() + "-alias",
+            Function.identity()));
+    }
 
     public void testNoDocs() throws IOException {
         try (Directory directory = newDirectory()) {
@@ -638,6 +653,49 @@ public class NestedAggregatorTests extends AggregatorTestCase {
         }
     }
 
+    public void testFieldAlias() throws IOException {
+        int numRootDocs = randomIntBetween(1, 20);
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+            NumberFieldMapper.NumberType.LONG);
+        fieldType.setName(VALUE_FIELD_NAME);
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < numRootDocs; i++) {
+                    List<Document> documents = new ArrayList<>();
+                    int numNestedDocs = randomIntBetween(0, 20);
+                    generateDocuments(documents, numNestedDocs, i, NESTED_OBJECT, VALUE_FIELD_NAME);
+
+                    Document document = new Document();
+                    document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)), IdFieldMapper.Defaults.FIELD_TYPE));
+                    document.add(new Field(TypeFieldMapper.NAME, "test",
+                        TypeFieldMapper.Defaults.FIELD_TYPE));
+                    document.add(sequenceIDFields.primaryTerm);
+                    documents.add(document);
+                    iw.addDocuments(documents);
+                }
+                iw.commit();
+            }
+
+            try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) {
+                NestedAggregationBuilder agg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation(
+                    max(MAX_AGG_NAME).field(VALUE_FIELD_NAME));
+
+                NestedAggregationBuilder aliasAgg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation(
+                    max(MAX_AGG_NAME).field(VALUE_FIELD_NAME + "-alias"));
+
+                Nested nested = search(newSearcher(indexReader, false, true),
+                    new MatchAllDocsQuery(), agg, fieldType);
+                Nested aliasNested = search(newSearcher(indexReader, false, true),
+                    new MatchAllDocsQuery(), aliasAgg, fieldType);
+
+                assertTrue(nested.getDocCount() > 0);
+                assertEquals(nested, aliasNested);
+            }
+        }
+    }
+
     private double generateMaxDocs(List<Document> documents, int numNestedDocs, int id, String path, String fieldName) {
         return DoubleStream.of(generateDocuments(documents, numNestedDocs, id, path, fieldName))
             .max().orElse(Double.NEGATIVE_INFINITY);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
index 36d6c6bd6e4..bc870bf4dca 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
@@ -40,7 +40,15 @@ import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.nested;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.reverseNested;
 
 public class ReverseNestedAggregatorTests extends AggregatorTestCase {
 
@@ -50,6 +58,15 @@ public class ReverseNestedAggregatorTests extends AggregatorTestCase {
     private static final String REVERSE_AGG_NAME = "reverseNestedAgg";
     private static final String MAX_AGG_NAME = "maxAgg";
 
+    /**
+     * For each provided field type, we also register an alias with name <field>-alias.
+     */
+    @Override
+    protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {
+        return Arrays.stream(fieldTypes).collect(Collectors.toMap(
+            ft -> ft.name() + "-alias",
+            Function.identity()));
+    }
 
     public void testNoDocs() throws IOException {
         try (Directory directory = newDirectory()) {
@@ -150,4 +167,63 @@ public class ReverseNestedAggregatorTests extends AggregatorTestCase {
         }
     }
 
+    public void testFieldAlias() throws IOException {
+        int numParentDocs = randomIntBetween(1, 20);
+
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+            NumberFieldMapper.NumberType.LONG);
+        fieldType.setName(VALUE_FIELD_NAME);
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < numParentDocs; i++) {
+                    List<Document> documents = new ArrayList<>();
+                    int numNestedDocs = randomIntBetween(0, 20);
+                    for (int nested = 0; nested < numNestedDocs; nested++) {
+                        Document document = new Document();
+                        document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)),
+                                IdFieldMapper.Defaults.NESTED_FIELD_TYPE));
+                        document.add(new Field(TypeFieldMapper.NAME, "__" + NESTED_OBJECT,
+                                TypeFieldMapper.Defaults.FIELD_TYPE));
+                        documents.add(document);
+                    }
+                    Document document = new Document();
+                    document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)),
+                            IdFieldMapper.Defaults.FIELD_TYPE));
+                    document.add(new Field(TypeFieldMapper.NAME, "test",
+                            TypeFieldMapper.Defaults.FIELD_TYPE));
+
+                    long value = randomNonNegativeLong() % 10000;
+                    document.add(new SortedNumericDocValuesField(VALUE_FIELD_NAME, value));
+                    document.add(SeqNoFieldMapper.SequenceIDFields.emptySeqID().primaryTerm);
+                    documents.add(document);
+                    iw.addDocuments(documents);
+                }
+                iw.commit();
+            }
+
+            try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) {
+
+                MaxAggregationBuilder maxAgg = max(MAX_AGG_NAME).field(VALUE_FIELD_NAME);
+                MaxAggregationBuilder aliasMaxAgg = max(MAX_AGG_NAME).field(VALUE_FIELD_NAME + "-alias");
+
+                NestedAggregationBuilder agg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation(
+                    reverseNested(REVERSE_AGG_NAME).subAggregation(maxAgg));
+                NestedAggregationBuilder aliasAgg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation(
+                    reverseNested(REVERSE_AGG_NAME).subAggregation(aliasMaxAgg));
+
+                Nested nested = search(newSearcher(indexReader, false, true),
+                        new MatchAllDocsQuery(), agg, fieldType);
+                Nested aliasNested = search(newSearcher(indexReader, false, true),
+                    new MatchAllDocsQuery(), aliasAgg, fieldType);
+
+                ReverseNested reverseNested = nested.getAggregations().get(REVERSE_AGG_NAME);
+                ReverseNested aliasReverseNested = aliasNested.getAggregations().get(REVERSE_AGG_NAME);
+
+                assertTrue(reverseNested.getDocCount() > 0);
+                assertEquals(reverseNested, aliasReverseNested);
+            }
+        }
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
index 8515f0a8994..70f9667ce7b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
@@ -55,7 +55,13 @@ import org.hamcrest.Matchers;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.search.aggregations.AggregationBuilders.significantTerms;
 
 public class SignificantTermsAggregatorTests extends AggregatorTestCase {
 
@@ -70,6 +76,16 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
         fieldType.setName("field");
     }
 
+    /**
+     * For each provided field type, we also register an alias with name <field>-alias.
+     */
+    @Override
+    protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {
+        return Arrays.stream(fieldTypes).collect(Collectors.toMap(
+            ft -> ft.name() + "-alias",
+            Function.identity()));
+    }
+
     public void testParsedAsFilter() throws IOException {
         IndexReader indexReader = new MultiReader();
         IndexSearcher indexSearcher = newSearcher(indexReader);
@@ -104,7 +120,7 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
         IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
         indexWriterConfig.setMaxBufferedDocs(100);
         indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment
-        
+
         try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) {
             addMixedTextDocs(textFieldType, w);
 
@@ -137,7 +153,7 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
                 assertNull(terms.getBucketByKey("odd"));
                 assertNull(terms.getBucketByKey("common"));
                 assertNotNull(terms.getBucketByKey("even"));
-                
+
                 // Search odd with regex includeexcludes
                 sigAgg.includeExclude(new IncludeExclude("o.d", null));
                 terms = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")), sigAgg, textFieldType);
@@ -149,7 +165,7 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
                 // Search with string-based includeexcludes
                 String oddStrings[] = new String[] {"odd", "weird"};
                 String evenStrings[] = new String[] {"even", "regular"};
-                
+
                 sigAgg.includeExclude(new IncludeExclude(oddStrings, evenStrings));
                 sigAgg.significanceHeuristic(SignificanceHeuristicTests.getRandomSignificanceheuristic());
                 terms = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")), sigAgg, textFieldType);
@@ -159,7 +175,7 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
                 assertNull(terms.getBucketByKey("common"));
                 assertNull(terms.getBucketByKey("even"));
                 assertNull(terms.getBucketByKey("regular"));
-                
+
                 sigAgg.includeExclude(new IncludeExclude(evenStrings, oddStrings));
                 terms = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")), sigAgg, textFieldType);
                 assertEquals(0, terms.getBuckets().size());
@@ -168,7 +184,7 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
                 assertNull(terms.getBucketByKey("common"));
                 assertNull(terms.getBucketByKey("even"));
                 assertNull(terms.getBucketByKey("regular"));
-                
+
             }
         }
     }
@@ -232,7 +248,7 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
             }
         }
     }
-    
+
     /**
      * Uses the significant terms aggregation on an index with unmapped field
      */
@@ -266,7 +282,57 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
 
             }
         }
-    }  
+    }
+
+    public void testFieldAlias() throws IOException {
+        TextFieldType textFieldType = new TextFieldType();
+        textFieldType.setName("text");
+        textFieldType.setFielddata(true);
+        textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer()));
+
+        IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+        indexWriterConfig.setMaxBufferedDocs(100);
+        indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment
+
+        try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) {
+            addMixedTextDocs(textFieldType, w);
+
+            SignificantTermsAggregationBuilder agg = significantTerms("sig_text").field("text");
+            SignificantTermsAggregationBuilder aliasAgg = significantTerms("sig_text").field("text-alias");
+
+            String executionHint = randomExecutionHint();
+            agg.executionHint(executionHint);
+            aliasAgg.executionHint(executionHint);
+
+            if (randomBoolean()) {
+                // Use a background filter which just happens to be same scope as whole-index.
+                QueryBuilder backgroundFilter = QueryBuilders.termsQuery("text", "common");
+                agg.backgroundFilter(backgroundFilter);
+                aliasAgg.backgroundFilter(backgroundFilter);
+            }
+
+            try (IndexReader reader = DirectoryReader.open(w)) {
+                assertEquals("test expects a single segment", 1, reader.leaves().size());
+                IndexSearcher searcher = new IndexSearcher(reader);
+
+                SignificantTerms evenTerms = searchAndReduce(searcher, new TermQuery(new Term("text", "even")),
+                    agg, textFieldType);
+                SignificantTerms aliasEvenTerms = searchAndReduce(searcher, new TermQuery(new Term("text", "even")),
+                    aliasAgg, textFieldType);
+
+                assertFalse(evenTerms.getBuckets().isEmpty());
+                assertEquals(evenTerms, aliasEvenTerms);
+
+                SignificantTerms oddTerms = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")),
+                    agg, textFieldType);
+                SignificantTerms aliasOddTerms = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")),
+                    aliasAgg, textFieldType);
+
+                assertFalse(oddTerms.getBuckets().isEmpty());
+                assertEquals(oddTerms, aliasOddTerms);
+            }
+        }
+    }
 
     private void addMixedTextDocs(TextFieldType textFieldType, IndexWriter w) throws IOException {
         for (int i = 0; i < 10; i++) {
@@ -284,7 +350,7 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
 
             w.addDocument(doc);
         }
-    }    
+    }
 
     private void addFields(Document doc, List<Field> createFields) {
         for (Field field : createFields) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
index ec15e2fc3e8..c63d5cb7d39 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
@@ -34,19 +34,34 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.bucket.sampler.Sampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTextAggregationBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.search.aggregations.AggregationBuilders.sampler;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.significantText;
 
 public class SignificantTextAggregatorTests extends AggregatorTestCase {
-  
-    
+
+    /**
+     * For each provided field type, we also register an alias with name <field>-alias.
+     */
+    @Override
+    protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {
+        return Arrays.stream(fieldTypes).collect(Collectors.toMap(
+            ft -> ft.name() + "-alias",
+            Function.identity()));
+    }
+
     /**
      * Uses the significant text aggregation to find the keywords in text fields
      */
@@ -59,22 +74,7 @@ public class SignificantTextAggregatorTests extends AggregatorTestCase {
         indexWriterConfig.setMaxBufferedDocs(100);
         indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment
         try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) {
-            for (int i = 0; i < 10; i++) {
-                Document doc = new Document();
-                StringBuilder text = new StringBuilder("common ");
-                if (i % 2 == 0) {
-                    text.append("odd ");
-                } else {                    
-                    text.append("even separator" + i + " duplicate duplicate duplicate duplicate duplicate duplicate ");
-                }
-
-                doc.add(new Field("text", text.toString(), textFieldType));
-                String json ="{ \"text\" : \"" + text.toString() + "\","+
-                             " \"json_only_field\" : \"" + text.toString() + "\"" +
-                            " }";
-                doc.add(new StoredField("_source", new BytesRef(json)));
-                w.addDocument(doc);
-            }
+            indexDocuments(w, textFieldType);
 
             SignificantTextAggregationBuilder sigAgg = new SignificantTextAggregationBuilder("sig_text", "text").filterDuplicateText(true);
             if(randomBoolean()){
@@ -82,37 +82,104 @@ public class SignificantTextAggregatorTests extends AggregatorTestCase {
             }
             SamplerAggregationBuilder aggBuilder = new SamplerAggregationBuilder("sampler")
                     .subAggregation(sigAgg);
-            
+
             try (IndexReader reader = DirectoryReader.open(w)) {
                 assertEquals("test expects a single segment", 1, reader.leaves().size());
                 IndexSearcher searcher = new IndexSearcher(reader);
-                                
+
                 // Search "odd" which should have no duplication
                 Sampler sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")), aggBuilder, textFieldType);
                 SignificantTerms terms = sampler.getAggregations().get("sig_text");
-                
+
                 assertNull(terms.getBucketByKey("even"));
-                assertNull(terms.getBucketByKey("duplicate"));                
-                assertNull(terms.getBucketByKey("common"));                
+                assertNull(terms.getBucketByKey("duplicate"));
+                assertNull(terms.getBucketByKey("common"));
                 assertNotNull(terms.getBucketByKey("odd"));
 
                 // Search "even" which will have duplication
                 sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "even")), aggBuilder, textFieldType);
                 terms = sampler.getAggregations().get("sig_text");
-                
+
                 assertNull(terms.getBucketByKey("odd"));
-                assertNull(terms.getBucketByKey("duplicate"));                
-                assertNull(terms.getBucketByKey("common"));    
+                assertNull(terms.getBucketByKey("duplicate"));
+                assertNull(terms.getBucketByKey("common"));
                 assertNull(terms.getBucketByKey("separator2"));
                 assertNull(terms.getBucketByKey("separator4"));
                 assertNull(terms.getBucketByKey("separator6"));
 
                 assertNotNull(terms.getBucketByKey("even"));
-            
+
             }
         }
     }
-    
+
+    public void testFieldAlias() throws IOException {
+        TextFieldType textFieldType = new TextFieldType();
+        textFieldType.setName("text");
+        textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer()));
+
+        IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+        indexWriterConfig.setMaxBufferedDocs(100);
+        indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment
+        try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) {
+            indexDocuments(w, textFieldType);
+
+            SignificantTextAggregationBuilder agg = significantText("sig_text", "text")
+                .filterDuplicateText(true);
+            SignificantTextAggregationBuilder aliasAgg = significantText("sig_text", "text-alias")
+                .filterDuplicateText(true);
+
+            if (randomBoolean()) {
+                List<String> sourceFieldNames = Arrays.asList(new String [] {"json_only_field"});
+                agg.sourceFieldNames(sourceFieldNames);
+                aliasAgg.sourceFieldNames(sourceFieldNames);
+            }
+
+            try (IndexReader reader = DirectoryReader.open(w)) {
+                assertEquals("test expects a single segment", 1, reader.leaves().size());
+                IndexSearcher searcher = new IndexSearcher(reader);
+
+                SamplerAggregationBuilder samplerAgg = sampler("sampler").subAggregation(agg);
+                SamplerAggregationBuilder aliasSamplerAgg = sampler("sampler").subAggregation(aliasAgg);
+
+                Sampler sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")), samplerAgg, textFieldType);
+                Sampler aliasSampler = searchAndReduce(searcher, new TermQuery(new Term("text", "odd")), aliasSamplerAgg, textFieldType);
+
+                SignificantTerms terms = sampler.getAggregations().get("sig_text");
+                SignificantTerms aliasTerms = aliasSampler.getAggregations().get("sig_text");
+                assertFalse(terms.getBuckets().isEmpty());
+                assertEquals(terms, aliasTerms);
+
+                sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "even")), samplerAgg, textFieldType);
+                aliasSampler = searchAndReduce(searcher, new TermQuery(new Term("text", "even")), aliasSamplerAgg, textFieldType);
+
+                terms = sampler.getAggregations().get("sig_text");
+                aliasTerms = aliasSampler.getAggregations().get("sig_text");
+                assertFalse(terms.getBuckets().isEmpty());
+                assertEquals(terms, aliasTerms);
+            }
+        }
+    }
+
+    private void indexDocuments(IndexWriter writer, TextFieldType textFieldType) throws IOException {
+        for (int i = 0; i < 10; i++) {
+            Document doc = new Document();
+            StringBuilder text = new StringBuilder("common ");
+            if (i % 2 == 0) {
+                text.append("odd ");
+            } else {
+                text.append("even separator" + i + " duplicate duplicate duplicate duplicate duplicate duplicate ");
+            }
+
+            doc.add(new Field("text", text.toString(), textFieldType));
+            String json ="{ \"text\" : \"" + text.toString() + "\","+
+                " \"json_only_field\" : \"" + text.toString() + "\"" +
+                " }";
+            doc.add(new StoredField("_source", new BytesRef(json)));
+            writer.addDocument(doc);
+        }
+    }
+
     /**
      * Test documents with arrays of text
      */
@@ -137,13 +204,13 @@ public class SignificantTextAggregatorTests extends AggregatorTestCase {
             sigAgg.sourceFieldNames(Arrays.asList(new String [] {"title", "text"}));
             try (IndexReader reader = DirectoryReader.open(w)) {
                 assertEquals("test expects a single segment", 1, reader.leaves().size());
-                IndexSearcher searcher = new IndexSearcher(reader);                                
+                IndexSearcher searcher = new IndexSearcher(reader);
                 searchAndReduce(searcher, new TermQuery(new Term("text", "foo")), sigAgg, textFieldType);
                 // No significant results to be found in this test - only checking we don't end up
                 // with the internal exception discovered in issue https://github.com/elastic/elasticsearch/issues/25029
             }
         }
     }
-    
-    
+
+
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index 4ff3c5fc5b4..d74e43327fd 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -809,12 +809,12 @@ public class TermsAggregatorTests extends AggregatorTestCase {
                 fieldType1.setHasDocValues(true);
 
                 MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
-                fieldType1.setName("another_long");
-                fieldType1.setHasDocValues(true);
+                fieldType2.setName("another_long");
+                fieldType2.setHasDocValues(true);
 
                 MappedFieldType fieldType3 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
-                fieldType1.setName("another_double");
-                fieldType1.setHasDocValues(true);
+                fieldType3.setName("another_double");
+                fieldType3.setHasDocValues(true);
                 try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
                     IndexSearcher indexSearcher = newIndexSearcher(indexReader);
                     ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG, ValueType.DOUBLE};
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index 1b4ae466bb3..b3a5df4dbfc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -18,12 +18,7 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
+import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
@@ -36,6 +31,14 @@ import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
+import org.hamcrest.core.IsNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@@ -61,6 +64,33 @@ public class SumIT extends AbstractNumericTestCase {
         return Collections.singleton(MetricAggScriptPlugin.class);
     }
 
+    @Override
+    public void setupSuiteScopeCluster() throws Exception {
+        super.setupSuiteScopeCluster();
+
+        // Create two indices and add the field 'route_length_miles' as an alias in
+        // one, and a concrete field in the other.
+        prepareCreate("old_index")
+            .addMapping("_doc",
+                "transit_mode", "type=keyword",
+                "distance", "type=double",
+                "route_length_miles", "type=alias,path=distance")
+            .get();
+        prepareCreate("new_index")
+            .addMapping("_doc",
+                "transit_mode", "type=keyword",
+                "route_length_miles", "type=double")
+            .get();
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(client().prepareIndex("old_index", "_doc").setSource("transit_mode", "train", "distance", 42.0));
+        builders.add(client().prepareIndex("old_index", "_doc").setSource("transit_mode", "bus", "distance", 50.5));
+        builders.add(client().prepareIndex("new_index", "_doc").setSource("transit_mode", "train", "route_length_miles", 100.2));
+
+        indexRandom(true, builders);
+        ensureSearchable();
+    }
+
     @Override
     public void testEmptyAggregation() throws Exception {
 
@@ -382,4 +412,54 @@ public class SumIT extends AbstractNumericTestCase {
         assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
                 .getMissCount(), equalTo(1L));
     }
+
+    public void testFieldAlias() {
+        SearchResponse response = client().prepareSearch("old_index", "new_index")
+            .addAggregation(sum("sum")
+                .field("route_length_miles"))
+            .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        Sum sum = response.getAggregations().get("sum");
+        assertThat(sum, IsNull.notNullValue());
+        assertThat(sum.getName(), equalTo("sum"));
+        assertThat(sum.getValue(), equalTo(192.7));
+    }
+
+     public void testFieldAliasInSubAggregation() {
+        SearchResponse response = client().prepareSearch("old_index", "new_index")
+            .addAggregation(terms("terms")
+                .field("transit_mode")
+                .subAggregation(sum("sum")
+                    .field("route_length_miles")))
+            .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        Terms terms = response.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getName(), equalTo("terms"));
+
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
+        assertThat(buckets.size(), equalTo(2));
+
+        Terms.Bucket bucket = buckets.get(0);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("train"));
+        assertThat(bucket.getDocCount(), equalTo(2L));
+
+        Sum sum = bucket.getAggregations().get("sum");
+        assertThat(sum, notNullValue());
+        assertThat(sum.getValue(), equalTo(142.2));
+
+        bucket = buckets.get(1);
+        assertThat(bucket, notNullValue());
+        assertThat(bucket.getKey(), equalTo("bus"));
+        assertThat(bucket.getDocCount(), equalTo(1L));
+
+        sum = bucket.getAggregations().get("sum");
+        assertThat(sum, notNullValue());
+        assertThat(sum.getValue(), equalTo(50.5));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
index b2a949ceeee..5124503fc03 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
@@ -26,10 +26,8 @@ import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngine;
@@ -344,8 +342,7 @@ public class ScriptedMetricAggregatorTests extends AggregatorTestCase {
      * is final and cannot be mocked
      */
     @Override
-    protected QueryShardContext queryShardContextMock(MapperService mapperService, final MappedFieldType[] fieldTypes,
-            CircuitBreakerService circuitBreakerService) {
+    protected QueryShardContext queryShardContextMock(MapperService mapperService) {
         MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS);
         Map<String, ScriptEngine> engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine);
         ScriptService scriptService =  new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
index ae65bc9f32c..b213ca785e2 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
@@ -258,4 +258,27 @@ public class ValuesSourceConfigTests extends ESSingleNodeTestCase {
             assertEquals(1, values.nextValue());
         }
     }
+
+
+    public void testFieldAlias() throws Exception {
+        IndexService indexService = createIndex("index", Settings.EMPTY, "type",
+            "field", "type=keyword", "alias", "type=alias,path=field");
+        client().prepareIndex("index", "type", "1")
+            .setSource("field", "value")
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+
+        try (Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
+            QueryShardContext context = indexService.newQueryShardContext(0, searcher.reader(), () -> 42L, null);
+            ValuesSourceConfig<ValuesSource.Bytes> config = ValuesSourceConfig.resolve(
+                context, ValueType.STRING, "alias", null, null, null, null);
+            ValuesSource.Bytes valuesSource = config.toValuesSource(context);
+
+            LeafReaderContext ctx = searcher.reader().leaves().get(0);
+            SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+            assertTrue(values.advanceExact(0));
+            assertEquals(1, values.docValueCount());
+            assertEquals(new BytesRef("value"), values.nextValue());
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index e5af22cd2ae..069c72c10b4 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -171,6 +171,106 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         }
     }
 
+    public void testFieldAlias() throws IOException {
+        XContentBuilder mappings = jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("text")
+                            .field("type", "text")
+                            .field("store", true)
+                            .field("term_vector", "with_positions_offsets")
+                        .endObject()
+                        .startObject("alias")
+                            .field("type", "alias")
+                            .field("path", "text")
+                        .endObject()
+                    .endObject()
+            .endObject()
+        .endObject();
+        assertAcked(prepareCreate("test").addMapping("type", mappings));
+
+        client().prepareIndex("test", "type", "1").setSource("text", "foo").get();
+        refresh();
+
+        for (String type : ALL_TYPES) {
+            HighlightBuilder builder = new HighlightBuilder()
+                .field(new Field("alias").highlighterType(type))
+                .requireFieldMatch(randomBoolean());
+            SearchResponse search = client().prepareSearch()
+                .setQuery(matchQuery("alias", "foo"))
+                .highlighter(builder)
+                .get();
+            assertHighlight(search, 0, "alias", 0, equalTo("<em>foo</em>"));
+        }
+    }
+
+    public void testFieldAliasWithSourceLookup() throws IOException {
+        XContentBuilder mappings = jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("text")
+                            .field("type", "text")
+                            .field("analyzer", "whitespace")
+                            .field("store", false)
+                            .field("term_vector", "with_positions_offsets")
+                        .endObject()
+                        .startObject("alias")
+                            .field("type", "alias")
+                            .field("path", "text")
+                        .endObject()
+                    .endObject()
+            .endObject()
+        .endObject();
+        assertAcked(prepareCreate("test").addMapping("type", mappings));
+
+        client().prepareIndex("test", "type", "1").setSource("text", "foo bar").get();
+        refresh();
+
+        for (String type : ALL_TYPES) {
+            HighlightBuilder builder = new HighlightBuilder()
+                .field(new Field("alias").highlighterType(type))
+                .requireFieldMatch(randomBoolean());
+            SearchResponse search = client().prepareSearch()
+                .setQuery(matchQuery("alias", "bar"))
+                .highlighter(builder)
+                .get();
+            assertHighlight(search, 0, "alias", 0, equalTo("foo <em>bar</em>"));
+        }
+    }
+
+    public void testFieldAliasWithWildcardField() throws IOException {
+        XContentBuilder mappings = jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("keyword")
+                            .field("type", "keyword")
+                        .endObject()
+                        .startObject("alias")
+                            .field("type", "alias")
+                            .field("path", "keyword")
+                        .endObject()
+                    .endObject()
+            .endObject()
+        .endObject();
+        assertAcked(prepareCreate("test").addMapping("type", mappings));
+
+        client().prepareIndex("test", "type", "1").setSource("keyword", "foo").get();
+        refresh();
+
+        HighlightBuilder builder = new HighlightBuilder()
+            .field(new Field("al*"))
+            .requireFieldMatch(false);
+        SearchResponse search = client().prepareSearch()
+            .setQuery(matchQuery("alias", "foo"))
+            .highlighter(builder)
+            .get();
+        assertHighlight(search, 0, "alias", 0, equalTo("<em>foo</em>"));
+    }
+
+
     public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOException {
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
diff --git a/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
new file mode 100644
index 00000000000..8440357758e
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.fieldcaps;
+
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+public class FieldCapabilitiesIT extends ESIntegTestCase {
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+
+        XContentBuilder oldIndexMapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("_doc")
+                    .startObject("properties")
+                        .startObject("distance")
+                            .field("type", "double")
+                        .endObject()
+                        .startObject("route_length_miles")
+                            .field("type", "alias")
+                            .field("path", "distance")
+                        .endObject()
+                        .startObject("playlist")
+                            .field("type", "text")
+                        .endObject()
+                        .startObject("secret_soundtrack")
+                            .field("type", "alias")
+                            .field("path", "playlist")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        assertAcked(prepareCreate("old_index").addMapping("_doc", oldIndexMapping));
+
+        XContentBuilder newIndexMapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("_doc")
+                    .startObject("properties")
+                        .startObject("distance")
+                            .field("type", "text")
+                        .endObject()
+                        .startObject("route_length_miles")
+                            .field("type", "double")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        assertAcked(prepareCreate("new_index").addMapping("_doc", newIndexMapping));
+    }
+
+    public static class FieldFilterPlugin extends Plugin implements MapperPlugin {
+        @Override
+        public Function<String, Predicate<String>> getFieldFilter() {
+            return index -> field -> !field.equals("playlist");
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(FieldFilterPlugin.class);
+    }
+
+    public void testFieldAlias() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("distance", "route_length_miles")
+            .execute().actionGet();
+
+        // Ensure the response has entries for both requested fields.
+        assertTrue(response.get().containsKey("distance"));
+        assertTrue(response.get().containsKey("route_length_miles"));
+
+        // Check the capabilities for the 'distance' field.
+        Map<String, FieldCapabilities> distance = response.getField("distance");
+        assertEquals(2, distance.size());
+
+        assertTrue(distance.containsKey("double"));
+        assertEquals(
+            new FieldCapabilities("distance", "double", true, true, new String[] {"old_index"}, null, null),
+            distance.get("double"));
+
+        assertTrue(distance.containsKey("text"));
+        assertEquals(
+            new FieldCapabilities("distance", "text", true, false, new String[] {"new_index"}, null, null),
+            distance.get("text"));
+
+        // Check the capabilities for the 'route_length_miles' alias.
+        Map<String, FieldCapabilities> routeLength = response.getField("route_length_miles");
+        assertEquals(1, routeLength.size());
+
+        assertTrue(routeLength.containsKey("double"));
+        assertEquals(
+            new FieldCapabilities("route_length_miles", "double", true, true),
+            routeLength.get("double"));
+    }
+
+    public void testFieldAliasWithWildcard() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("route*")
+            .execute().actionGet();
+
+        assertEquals(1, response.get().size());
+        assertTrue(response.get().containsKey("route_length_miles"));
+    }
+
+    public void testFieldAliasFiltering() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields(
+            "secret-soundtrack", "route_length_miles")
+            .execute().actionGet();
+        assertEquals(1, response.get().size());
+        assertTrue(response.get().containsKey("route_length_miles"));
+    }
+
+    public void testFieldAliasFilteringWithWildcard() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps()
+            .setFields("distance", "secret*")
+            .execute().actionGet();
+        assertEquals(1, response.get().size());
+        assertTrue(response.get().containsKey("distance"));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
index 2e9426bbe68..2126e0e94eb 100644
--- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -46,7 +47,12 @@ import org.elasticsearch.search.lookup.FieldLookup;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.ReadableDateTime;
 import org.joda.time.base.BaseDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
@@ -913,6 +919,163 @@ public class SearchFieldsIT extends ESIntegTestCase {
         }
     }
 
+    public void testDocValueFieldsWithFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("_source")
+                        .field("enabled", false)
+                    .endObject()
+                    .startObject("properties")
+                        .startObject("text_field")
+                            .field("type", "text")
+                            .field("fielddata", true)
+                        .endObject()
+                        .startObject("date_field")
+                            .field("type", "date")
+                            .field("format", "yyyy-MM-dd")
+                        .endObject()
+                        .startObject("text_field_alias")
+                            .field("type", "alias")
+                            .field("path", "text_field")
+                        .endObject()
+                        .startObject("date_field_alias")
+                            .field("type", "alias")
+                            .field("path", "date_field")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        assertAcked(prepareCreate("test").addMapping("type", mapping));
+        ensureGreen("test");
+
+        DateTime date = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
+        DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd");
+
+        index("test", "type", "1", "text_field", "foo", "date_field", formatter.print(date));
+        refresh("test");
+
+        SearchRequestBuilder builder = client().prepareSearch().setQuery(matchAllQuery())
+                .addDocValueField("text_field_alias")
+                .addDocValueField("date_field_alias", "use_field_mapping")
+                .addDocValueField("date_field");
+        SearchResponse searchResponse = builder.execute().actionGet();
+
+        assertNoFailures(searchResponse);
+        assertHitCount(searchResponse, 1);
+        SearchHit hit = searchResponse.getHits().getAt(0);
+
+        Map<String, DocumentField> fields = hit.getFields();
+        assertThat(fields.keySet(), equalTo(newHashSet("text_field_alias", "date_field_alias", "date_field")));
+
+        DocumentField textFieldAlias = fields.get("text_field_alias");
+        assertThat(textFieldAlias.getName(), equalTo("text_field_alias"));
+        assertThat(textFieldAlias.getValue(), equalTo("foo"));
+
+        DocumentField dateFieldAlias = fields.get("date_field_alias");
+        assertThat(dateFieldAlias.getName(), equalTo("date_field_alias"));
+        assertThat(dateFieldAlias.getValue(),
+            equalTo("1990-12-29"));
+
+        DocumentField dateField = fields.get("date_field");
+        assertThat(dateField.getName(), equalTo("date_field"));
+
+        ReadableDateTime fetchedDate = dateField.getValue();
+        assertThat(fetchedDate, equalTo(date));
+    }
+
+
+    public void testStoredFieldsWithFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field1")
+                            .field("type", "text")
+                            .field("store", true)
+                        .endObject()
+                        .startObject("field2")
+                            .field("type", "text")
+                            .field("store", false)
+                        .endObject()
+                        .startObject("field1-alias")
+                            .field("type", "alias")
+                            .field("path", "field1")
+                        .endObject()
+                        .startObject("field2-alias")
+                            .field("type", "alias")
+                            .field("path", "field2")
+                        .endObject()
+                    .endObject()
+            .endObject()
+        .endObject();
+        assertAcked(prepareCreate("test").addMapping("type", mapping));
+
+        index("test", "type", "1", "field1", "value1", "field2", "value2");
+        refresh("test");
+
+        SearchResponse searchResponse = client().prepareSearch()
+            .setQuery(matchAllQuery())
+            .addStoredField("field1-alias")
+            .addStoredField("field2-alias")
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        SearchHit hit = searchResponse.getHits().getAt(0);
+        assertEquals(1, hit.getFields().size());
+        assertTrue(hit.getFields().containsKey("field1-alias"));
+
+        DocumentField field = hit.getFields().get("field1-alias");
+        assertThat(field.getValue().toString(), equalTo("value1"));
+    }
+
+    public void testWildcardStoredFieldsWithFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field1")
+                            .field("type", "text")
+                            .field("store", true)
+                        .endObject()
+                        .startObject("field2")
+                            .field("type", "text")
+                            .field("store", false)
+                        .endObject()
+                        .startObject("field1-alias")
+                            .field("type", "alias")
+                            .field("path", "field1")
+                        .endObject()
+                        .startObject("field2-alias")
+                            .field("type", "alias")
+                            .field("path", "field2")
+                        .endObject()
+                    .endObject()
+            .endObject()
+        .endObject();
+        assertAcked(prepareCreate("test").addMapping("type", mapping));
+
+        index("test", "type", "1", "field1", "value1", "field2", "value2");
+        refresh("test");
+
+        SearchResponse searchResponse = client().prepareSearch()
+            .setQuery(matchAllQuery())
+            .addStoredField("field*")
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        SearchHit hit = searchResponse.getHits().getAt(0);
+        assertEquals(2, hit.getFields().size());
+        assertTrue(hit.getFields().containsKey("field1"));
+        assertTrue(hit.getFields().containsKey("field1-alias"));
+
+        DocumentField field = hit.getFields().get("field1");
+        assertThat(field.getValue().toString(), equalTo("value1"));
+
+        DocumentField fieldAlias = hit.getFields().get("field1-alias");
+        assertThat(fieldAlias.getValue().toString(), equalTo("value1"));
+    }
+
     public void testLoadMetadata() throws Exception {
         assertAcked(prepareCreate("test"));
 
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java
index 7906165b090..2ff7d0c1383 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java
@@ -24,8 +24,6 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -58,10 +56,10 @@ public class GeoPolygonIT extends ESIntegTestCase {
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0,
                 Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("properties").startObject("location").field("type", "geo_point");
-        xContentBuilder.endObject().endObject().endObject().endObject();
-        assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder));
+
+        assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", "location",
+            "type=geo_point", "alias",
+            "type=alias,path=location"));
         ensureGreen();
 
         indexRandom(true, client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
@@ -132,4 +130,17 @@ public class GeoPolygonIT extends ESIntegTestCase {
             assertThat(hit.getId(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5")));
         }
     }
+
+    public void testFieldAlias() {
+        List<GeoPoint> points = new ArrayList<>();
+        points.add(new GeoPoint(40.7, -74.0));
+        points.add(new GeoPoint(40.7, -74.1));
+        points.add(new GeoPoint(40.8, -74.1));
+        points.add(new GeoPoint(40.8, -74.0));
+        points.add(new GeoPoint(40.7, -74.0));
+        SearchResponse searchResponse = client().prepareSearch("test") // from NY
+            .setQuery(boolQuery().must(geoPolygonQuery("alias", points)))
+            .execute().actionGet();
+        assertHitCount(searchResponse, 4);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index d3a31f12c57..6f204796e41 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -19,7 +19,10 @@
 
 package org.elasticsearch.search.geo;
 
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
 import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
 import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
@@ -27,20 +30,16 @@ import org.elasticsearch.common.geo.builders.LineStringBuilder;
 import org.elasticsearch.common.geo.builders.PolygonBuilder;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.locationtech.spatial4j.shape.Rectangle;
-import org.locationtech.jts.geom.Coordinate;
-
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.GeoShapeQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.spatial4j.shape.Rectangle;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -503,4 +502,33 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
 
         assertEquals(2, response.getHits().getTotalHits());
     }
+
+    public void testFieldAlias() throws IOException {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("location")
+                        .field("type", "geo_shape")
+                        .field("tree", randomBoolean() ? "quadtree" : "geohash")
+                    .endObject()
+                    .startObject("alias")
+                        .field("type", "alias")
+                        .field("path", "location")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject();
+
+        createIndex("test", Settings.EMPTY, "type", mapping);
+
+        ShapeBuilder shape = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.MULTIPOINT);
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("location", shape).endObject())
+            .setRefreshPolicy(IMMEDIATE).get();
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(geoShapeQuery("alias", shape))
+            .execute().actionGet();
+        assertEquals(1, response.getHits().getTotalHits());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
new file mode 100644
index 00000000000..dfdbef1c3d5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.lookup;
+
+import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import static org.mockito.AdditionalAnswers.returnsFirstArg;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class LeafDocLookupTests extends ESTestCase {
+    private ScriptDocValues<?> docValues;
+    private LeafDocLookup docLookup;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+
+        MappedFieldType fieldType = mock(MappedFieldType.class);
+        when(fieldType.name()).thenReturn("field");
+        when(fieldType.valueForDisplay(anyObject())).then(returnsFirstArg());
+
+        MapperService mapperService = mock(MapperService.class);
+        when(mapperService.fullName("field")).thenReturn(fieldType);
+        when(mapperService.fullName("alias")).thenReturn(fieldType);
+
+        docValues = mock(ScriptDocValues.class);
+
+        AtomicFieldData atomicFieldData = mock(AtomicFieldData.class);
+        doReturn(docValues).when(atomicFieldData).getScriptValues();
+
+        IndexFieldData<?> fieldData = mock(IndexFieldData.class);
+        when(fieldData.getFieldName()).thenReturn("field");
+        doReturn(atomicFieldData).when(fieldData).load(anyObject());
+
+        docLookup = new LeafDocLookup(mapperService,
+            ignored -> fieldData,
+            new String[] { "type" },
+            null);
+    }
+
+    public void testBasicLookup() {
+        ScriptDocValues<?> fetchedDocValues = docLookup.get("field");
+        assertEquals(docValues, fetchedDocValues);
+    }
+
+    public void testLookupWithFieldAlias() {
+        ScriptDocValues<?> fetchedDocValues = docLookup.get("alias");
+        assertEquals(docValues, fetchedDocValues);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java
new file mode 100644
index 00000000000..1195893a28a
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.lookup;
+
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.AdditionalAnswers.returnsFirstArg;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class LeafFieldsLookupTests extends ESTestCase {
+    private LeafFieldsLookup fieldsLookup;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+
+        MappedFieldType fieldType = mock(MappedFieldType.class);
+        when(fieldType.name()).thenReturn("field");
+        when(fieldType.valueForDisplay(anyObject())).then(returnsFirstArg());
+
+        MapperService mapperService = mock(MapperService.class);
+        when(mapperService.fullName("field")).thenReturn(fieldType);
+        when(mapperService.fullName("alias")).thenReturn(fieldType);
+
+        FieldInfo mockFieldInfo = new FieldInfo("field", 1, false, false, true,
+            IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, false);
+
+        LeafReader leafReader = mock(LeafReader.class);
+        doAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            StoredFieldVisitor visitor = (StoredFieldVisitor) args[1];
+            visitor.doubleField(mockFieldInfo, 2.718);
+            return null;
+        }).when(leafReader).document(anyInt(), any(StoredFieldVisitor.class));
+
+        fieldsLookup = new LeafFieldsLookup(mapperService,
+            new String[] { "type" },
+            leafReader);
+    }
+
+    public void testBasicLookup() {
+        FieldLookup fieldLookup = (FieldLookup) fieldsLookup.get("field");
+        assertEquals("field", fieldLookup.fieldType().name());
+
+        List<Object> values = fieldLookup.getValues();
+        assertNotNull(values);
+        assertEquals(1, values.size());
+        assertEquals(2.718, values.get(0));
+    }
+
+    public void testLookupWithFieldAlias() {
+        FieldLookup fieldLookup = (FieldLookup) fieldsLookup.get("alias");
+        assertEquals("field", fieldLookup.fieldType().name());
+
+        List<Object> values = fieldLookup.getValues();
+        assertNotNull(values);
+        assertEquals(1, values.size());
+        assertEquals(2.718, values.get(0));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/server/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
index 81f48aac074..dedd0f03664 100644
--- a/server/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
+++ b/server/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder;
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -337,6 +338,36 @@ public class MoreLikeThisIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 0L);
     }
 
+    public void testMoreLikeThisWithFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("_doc")
+                .startObject("properties")
+                    .startObject("text")
+                        .field("type", "text")
+                    .endObject()
+                    .startObject("alias")
+                        .field("type", "alias")
+                        .field("path", "text")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject();
+
+        assertAcked(prepareCreate("test").addMapping("_doc", mapping));
+        ensureGreen();
+
+        index("test", "_doc", "1", "text", "lucene");
+        index("test", "_doc", "2", "text", "lucene release");
+        refresh();
+
+        Item item = new Item("test", "_doc", "1");
+        QueryBuilder query = QueryBuilders.moreLikeThisQuery(new String[] {"alias"}, null, new Item[] {item})
+            .minTermFreq(1)
+            .minDocFreq(1);
+        SearchResponse response = client().prepareSearch().setQuery(query).get();
+        assertHitCount(response, 1L);
+    }
+
     public void testSimpleMoreLikeInclude() throws Exception {
         logger.info("Creating index test");
         assertAcked(prepareCreate("test").addMapping("type1",
diff --git a/server/src/test/java/org/elasticsearch/search/query/ExistsIT.java b/server/src/test/java/org/elasticsearch/search/query/ExistsIT.java
index 6e4a1b7d618..45910044d49 100644
--- a/server/src/test/java/org/elasticsearch/search/query/ExistsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/ExistsIT.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -141,4 +142,89 @@ public class ExistsIT extends ESIntegTestCase {
             }
         }
     }
+
+    public void testFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("bar")
+                            .field("type", "long")
+                        .endObject()
+                        .startObject("foo")
+                            .field("type", "object")
+                            .startObject("properties")
+                                .startObject("bar")
+                                    .field("type", "double")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject("foo-bar")
+                            .field("type", "alias")
+                            .field("path", "foo.bar")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        assertAcked(prepareCreate("idx").addMapping("type", mapping));
+        ensureGreen("idx");
+
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("idx", "type").setSource(emptyMap()));
+        indexRequests.add(client().prepareIndex("idx", "type").setSource(emptyMap()));
+        indexRequests.add(client().prepareIndex("idx", "type").setSource("bar", 3));
+        indexRequests.add(client().prepareIndex("idx", "type").setSource("foo", singletonMap("bar", 2.718)));
+        indexRequests.add(client().prepareIndex("idx", "type").setSource("foo", singletonMap("bar", 6.283)));
+        indexRandom(true, false, indexRequests);
+
+        Map<String, Integer> expected = new LinkedHashMap<>();
+        expected.put("foo.bar", 2);
+        expected.put("foo-bar", 2);
+        expected.put("foo*", 2);
+        expected.put("*bar", 3);
+
+        for (Map.Entry<String, Integer> entry : expected.entrySet()) {
+            String fieldName = entry.getKey();
+            int expectedCount = entry.getValue();
+
+            SearchResponse response = client().prepareSearch("idx")
+                .setQuery(QueryBuilders.existsQuery(fieldName))
+                .get();
+            assertSearchResponse(response);
+            assertHitCount(response, expectedCount);
+        }
+    }
+
+    public void testFieldAliasWithNoDocValues() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("foo")
+                            .field("type", "long")
+                            .field("doc_values", false)
+                        .endObject()
+                        .startObject("foo-alias")
+                            .field("type", "alias")
+                            .field("path", "foo")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        assertAcked(prepareCreate("idx").addMapping("type", mapping));
+        ensureGreen("idx");
+
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("idx", "type").setSource(emptyMap()));
+        indexRequests.add(client().prepareIndex("idx", "type").setSource(emptyMap()));
+        indexRequests.add(client().prepareIndex("idx", "type").setSource("foo", 3));
+        indexRequests.add(client().prepareIndex("idx", "type").setSource("foo", 43));
+        indexRandom(true, false, indexRequests);
+
+        SearchResponse response = client().prepareSearch("idx")
+            .setQuery(QueryBuilders.existsQuery("foo-alias"))
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 2);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
index 7145f9db2db..5caab8c9dfe 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
@@ -51,6 +51,7 @@ import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
@@ -376,6 +377,70 @@ public class QueryStringIT extends ESIntegTestCase {
                 containsString("field expansion matches too many fields, limit: 1024, got: 1025"));
     }
 
+    public void testFieldAlias() throws Exception {
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("test", "_doc", "1").setSource("f3", "text", "f2", "one"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "2").setSource("f3", "value", "f2", "two"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "3").setSource("f3", "another value", "f2", "three"));
+        indexRandom(true, false, indexRequests);
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(queryStringQuery("value").field("f3_alias"))
+            .execute().actionGet();
+
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertHits(response.getHits(), "2", "3");
+    }
+
+    public void testFieldAliasWithEmbeddedFieldNames() throws Exception {
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("test", "_doc", "1").setSource("f3", "text", "f2", "one"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "2").setSource("f3", "value", "f2", "two"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "3").setSource("f3", "another value", "f2", "three"));
+        indexRandom(true, false, indexRequests);
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(queryStringQuery("f3_alias:value AND f2:three"))
+            .execute().actionGet();
+
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "3");
+    }
+
+    public void testFieldAliasWithWildcardField() throws Exception {
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("test", "_doc", "1").setSource("f3", "text", "f2", "one"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "2").setSource("f3", "value", "f2", "two"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "3").setSource("f3", "another value", "f2", "three"));
+        indexRandom(true, false, indexRequests);
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(queryStringQuery("value").field("f3_*"))
+            .execute().actionGet();
+
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertHits(response.getHits(), "2", "3");
+    }
+
+    public void testFieldAliasOnDisallowedFieldType() throws Exception {
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("test", "_doc", "1").setSource("f3", "text", "f2", "one"));
+        indexRandom(true, false, indexRequests);
+
+        // The wildcard field matches aliases for both a text and boolean field.
+        // By default, the boolean field should be ignored when building the query.
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(queryStringQuery("text").field("f*_alias"))
+            .execute().actionGet();
+
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "1");
+    }
+
     private void assertHits(SearchHits hits, String... ids) {
         assertThat(hits.getTotalHits(), equalTo((long) ids.length));
         Set<String> hitIds = new HashSet<>();
diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 1694f86c53e..860c3e074f3 100644
--- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.query;
 
+import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.util.English;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -26,13 +27,16 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.Operator;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
@@ -1858,4 +1862,78 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 1);
     }
 
+    public void testNestedQueryWithFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("_doc")
+                    .startObject("properties")
+                        .startObject("section")
+                            .field("type", "nested")
+                            .startObject("properties")
+                                .startObject("distance")
+                                    .field("type", "long")
+                                .endObject()
+                                .startObject("route_length_miles")
+                                    .field("type", "alias")
+                                    .field("path", "section.distance")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        assertAcked(prepareCreate("index").addMapping("_doc", mapping));
+
+        XContentBuilder source = XContentFactory.jsonBuilder().startObject()
+            .startObject("section")
+                .field("distance", 42)
+            .endObject()
+        .endObject();
+
+        index("index", "_doc", "1", source);
+        refresh();
+
+        QueryBuilder nestedQuery = QueryBuilders.nestedQuery("section",
+            QueryBuilders.termQuery("section.route_length_miles", 42),
+            ScoreMode.Max);
+        SearchResponse searchResponse = client().prepareSearch("index").setQuery(nestedQuery).get();
+        assertHitCount(searchResponse, 1);
+    }
+
+   public void testFieldAliasesForMetaFields() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("id-alias")
+                            .field("type", "alias")
+                            .field("path", "_id")
+                        .endObject()
+                        .startObject("routing-alias")
+                            .field("type", "alias")
+                            .field("path", "_routing")
+                        .endObject()
+                    .endObject()
+            .endObject()
+        .endObject();
+        assertAcked(prepareCreate("test").addMapping("type", mapping));
+
+        IndexRequestBuilder indexRequest = client().prepareIndex("test", "type")
+            .setId("1")
+            .setRouting("custom")
+            .setSource("field", "value");
+        indexRandom(true, false, indexRequest);
+
+        SearchResponse searchResponse = client().prepareSearch()
+            .setQuery(termQuery("routing-alias", "custom"))
+            .addDocValueField("id-alias")
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        SearchHit hit = searchResponse.getHits().getAt(0);
+        assertEquals(2, hit.getFields().size());
+        assertTrue(hit.getFields().containsKey("id-alias"));
+
+        DocumentField field = hit.getFields().get("id-alias");
+        assertThat(field.getValue().toString(), equalTo("1"));
+   }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
index d7db62b6143..5176c327ac7 100644
--- a/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
@@ -58,6 +58,7 @@ import java.util.concurrent.ExecutionException;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
@@ -585,6 +586,67 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
                 containsString("field expansion matches too many fields, limit: 1024, got: 1025"));
     }
 
+    public void testFieldAlias() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        assertAcked(prepareCreate("test").setSource(indexBody, XContentType.JSON));
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("test", "_doc", "1").setSource("f3", "text", "f2", "one"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "2").setSource("f3", "value", "f2", "two"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "3").setSource("f3", "another value", "f2", "three"));
+        indexRandom(true, false, indexRequests);
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(simpleQueryStringQuery("value").field("f3_alias"))
+            .execute().actionGet();
+
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertHits(response.getHits(), "2", "3");
+    }
+
+    public void testFieldAliasWithWildcardField() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        assertAcked(prepareCreate("test").setSource(indexBody, XContentType.JSON));
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("test", "_doc", "1").setSource("f3", "text", "f2", "one"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "2").setSource("f3", "value", "f2", "two"));
+        indexRequests.add(client().prepareIndex("test", "_doc", "3").setSource("f3", "another value", "f2", "three"));
+        indexRandom(true, false, indexRequests);
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(simpleQueryStringQuery("value").field("f3_*"))
+            .execute().actionGet();
+
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertHits(response.getHits(), "2", "3");
+    }
+
+
+    public void testFieldAliasOnDisallowedFieldType() throws Exception {
+        String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
+        assertAcked(prepareCreate("test").setSource(indexBody, XContentType.JSON));
+        ensureGreen("test");
+
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("test", "_doc", "1").setSource("f3", "text", "f2", "one"));
+        indexRandom(true, false, indexRequests);
+
+        // The wildcard field matches aliases for both a text and boolean field.
+        // By default, the boolean field should be ignored when building the query.
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(queryStringQuery("text").field("f*_alias"))
+            .execute().actionGet();
+
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "1");
+    }
+
     private void assertHits(SearchHits hits, String... ids) {
         assertThat(hits.getTotalHits(), equalTo((long) ids.length));
         Set<String> hitIds = new HashSet<>();
diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index 3af370326f5..ff0196aacdf 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -40,6 +40,7 @@ import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.Matchers;
@@ -1572,4 +1573,60 @@ public class FieldSortIT extends ESIntegTestCase {
             }
         }
     }
+
+    public void testFieldAlias() throws Exception {
+        // Create two indices and add the field 'route_length_miles' as an alias in
+        // one, and a concrete field in the other.
+        assertAcked(prepareCreate("old_index")
+            .addMapping("_doc", "distance", "type=double", "route_length_miles", "type=alias,path=distance"));
+        assertAcked(prepareCreate("new_index")
+            .addMapping("_doc", "route_length_miles", "type=double"));
+        ensureGreen("old_index", "new_index");
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(client().prepareIndex("old_index", "_doc").setSource("distance", 42.0));
+        builders.add(client().prepareIndex("old_index", "_doc").setSource("distance", 50.5));
+        builders.add(client().prepareIndex("new_index", "_doc").setSource("route_length_miles", 100.2));
+        indexRandom(true, true, builders);
+
+        SearchResponse response = client().prepareSearch()
+            .setQuery(matchAllQuery())
+            .setSize(builders.size())
+            .addSort(SortBuilders.fieldSort("route_length_miles"))
+            .execute().actionGet();
+        SearchHits hits = response.getHits();
+
+        assertEquals(3, hits.getHits().length);
+        assertEquals(42.0, hits.getAt(0).getSortValues()[0]);
+        assertEquals(50.5, hits.getAt(1).getSortValues()[0]);
+        assertEquals(100.2, hits.getAt(2).getSortValues()[0]);
+    }
+
+    public void testFieldAliasesWithMissingValues() throws Exception {
+        // Create two indices and add the field 'route_length_miles' as an alias in
+        // one, and a concrete field in the other.
+        assertAcked(prepareCreate("old_index")
+            .addMapping("_doc", "distance", "type=double", "route_length_miles", "type=alias,path=distance"));
+        assertAcked(prepareCreate("new_index")
+            .addMapping("_doc", "route_length_miles", "type=double"));
+        ensureGreen("old_index", "new_index");
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(client().prepareIndex("old_index", "_doc").setSource("distance", 42.0));
+        builders.add(client().prepareIndex("old_index", "_doc").setSource(Collections.emptyMap()));
+        builders.add(client().prepareIndex("new_index", "_doc").setSource("route_length_miles", 100.2));
+        indexRandom(true, true, builders);
+
+        SearchResponse response = client().prepareSearch()
+            .setQuery(matchAllQuery())
+            .setSize(builders.size())
+            .addSort(SortBuilders.fieldSort("route_length_miles").missing(120.3))
+            .execute().actionGet();
+        SearchHits hits = response.getHits();
+
+        assertEquals(3, hits.getHits().length);
+        assertEquals(42.0, hits.getAt(0).getSortValues()[0]);
+        assertEquals(100.2, hits.getAt(1).getSortValues()[0]);
+        assertEquals(120.3, hits.getAt(2).getSortValues()[0]);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
index 00a287f0252..9d1f01fe328 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
@@ -160,7 +160,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
                     indexSettings);
             MapperService mapperService = mock(MapperService.class);
             ScriptService scriptService = mock(ScriptService.class);
-            MappedFieldType fieldType = mockFieldType();
+            MappedFieldType fieldType = mockFieldType(suggestionBuilder.field());
             boolean fieldTypeSearchAnalyzerSet = randomBoolean();
             if (fieldTypeSearchAnalyzerSet) {
                 NamedAnalyzer searchAnalyzer = new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer());
@@ -211,8 +211,10 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
      */
     protected abstract void assertSuggestionContext(SB builder, SuggestionContext context) throws IOException;
 
-    protected MappedFieldType mockFieldType() {
-        return mock(MappedFieldType.class);
+    protected MappedFieldType mockFieldType(String fieldName) {
+        MappedFieldType fieldType = mock(MappedFieldType.class);
+        when(fieldType.name()).thenReturn(fieldName);
+        return fieldType;
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
index a3fff7f9d5b..0c4c6e510eb 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.suggest;
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-
 import org.apache.lucene.analysis.TokenStreamToAutomaton;
 import org.apache.lucene.search.suggest.document.ContextSuggestField;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -36,6 +35,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.plugins.Plugin;
@@ -1161,6 +1161,32 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         assertSuggestions("foo", prefix, "suggester10", "suggester9", "suggester8", "suggester7", "suggester6");
     }
 
+    public void testSuggestWithFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject(TYPE)
+                    .startObject("properties")
+                        .startObject(FIELD)
+                            .field("type", "completion")
+                        .endObject()
+                        .startObject("alias")
+                            .field("type", "alias")
+                            .field("path", FIELD)
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping));
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(client().prepareIndex(INDEX, TYPE).setSource(FIELD, "apple"));
+        builders.add(client().prepareIndex(INDEX, TYPE).setSource(FIELD, "mango"));
+        builders.add(client().prepareIndex(INDEX, TYPE).setSource(FIELD, "papaya"));
+        indexRandom(true, false, builders);
+
+        CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("alias").text("app");
+        assertSuggestions("suggestion", suggestionBuilder, "apple");
+    }
 
     public static boolean isReservedChar(char c) {
         switch (c) {
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
index 677cc4163cc..aaeaadd4c9f 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
@@ -979,6 +979,35 @@ public class SuggestSearchIT extends ESIntegTestCase {
         // assertThat(total, lessThan(1000L)); // Takes many seconds without fix - just for debugging
     }
 
+    public void testSuggestWithFieldAlias() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("text")
+                            .field("type", "keyword")
+                        .endObject()
+                        .startObject("alias")
+                            .field("type", "alias")
+                            .field("path", "text")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+        assertAcked(prepareCreate("test").addMapping("type", mapping));
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(client().prepareIndex("test", "type").setSource("text", "apple"));
+        builders.add(client().prepareIndex("test", "type").setSource("text", "mango"));
+        builders.add(client().prepareIndex("test", "type").setSource("text", "papaya"));
+        indexRandom(true, false, builders);
+
+        TermSuggestionBuilder termSuggest = termSuggestion("alias").text("appple");
+
+        Suggest searchSuggest = searchSuggest("suggestion", termSuggest);
+        assertSuggestion(searchSuggest, 0, "suggestion", "apple");
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singleton(DummyTemplatePlugin.class);
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
index 3a7451e78fb..f9b252f0e13 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
@@ -31,6 +31,7 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -38,10 +39,10 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.mapper.CompletionFieldMapper.CompletionFieldType;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
@@ -74,8 +75,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(jsonBuilder()
                         .startObject()
@@ -95,7 +95,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                         .endArray()
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertContextSuggestFields(fields, 7);
     }
 
@@ -113,8 +113,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(jsonBuilder()
                         .startObject()
@@ -129,7 +128,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                         .endArray()
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertContextSuggestFields(fields, 3);
     }
 
@@ -147,8 +146,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(jsonBuilder()
                         .startObject()
@@ -163,7 +161,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                         .endArray()
                         .endObject()),
             XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertContextSuggestFields(fields, 3);
     }
 
@@ -181,8 +179,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(jsonBuilder()
                         .startObject()
@@ -197,7 +194,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                         .endArray()
                         .endObject()),
             XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertContextSuggestFields(fields, 3);
     }
 
@@ -247,8 +244,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(jsonBuilder()
                         .startObject()
@@ -261,7 +257,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                         .endObject()
                         .endObject()),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertContextSuggestFields(fields, 3);
     }
 
@@ -279,8 +275,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
                 .bytes(jsonBuilder()
                         .startObject()
@@ -293,7 +288,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                         .endObject()
                         .endObject()),
             XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertContextSuggestFields(fields, 3);
     }
 
@@ -345,8 +340,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject());
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         XContentBuilder builder = jsonBuilder()
                 .startObject()
                 .startArray("completion")
@@ -362,7 +356,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject();
         ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference.bytes(builder),
                 XContentType.JSON));
-        IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
+        IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
         assertContextSuggestFields(fields, 3);
     }
 
@@ -698,7 +692,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testUnknownQueryContextParsing() throws Exception {
-        String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1")
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("completion")
                 .field("type", "completion")
                 .startArray("contexts")
@@ -712,11 +706,10 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endArray()
                 .endObject().endObject()
-                .endObject().endObject());
+                .endObject().endObject();
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        CompletionFieldType completionFieldType = (CompletionFieldType) fieldMapper.fieldType();
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type1", mapping).mapperService();
+        CompletionFieldType completionFieldType = (CompletionFieldType) mapperService.fullName("completion");
 
         Exception e = expectThrows(IllegalArgumentException.class, () -> completionFieldType.getContextMappings().get("brand"));
         assertEquals("Unknown context name [brand], must be one of [ctx, type]", e.getMessage());
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
index 37fdb7e0aa0..88e6ce64666 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
@@ -164,8 +164,9 @@ public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTe
     }
 
     @Override
-    protected MappedFieldType mockFieldType() {
+    protected MappedFieldType mockFieldType(String fieldName) {
         CompletionFieldType completionFieldType = new CompletionFieldType();
+        completionFieldType.setName(fieldName);
         completionFieldType.setContextMappings(new ContextMappings(contextMappings));
         return completionFieldType;
     }
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
index 2d179f3dbe6..56ff157ec71 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
@@ -20,16 +20,14 @@
 package org.elasticsearch.search.suggest.completion;
 
 import org.apache.lucene.index.IndexableField;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.search.suggest.completion.context.ContextBuilder;
@@ -50,7 +48,7 @@ import static org.hamcrest.Matchers.isIn;
 public class GeoContextMappingTests extends ESSingleNodeTestCase {
 
     public void testIndexingWithNoContexts() throws Exception {
-        String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1")
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("completion")
                 .field("type", "completion")
                 .startArray("contexts")
@@ -60,13 +58,12 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endArray()
                 .endObject().endObject()
-                .endObject().endObject());
+                .endObject().endObject();
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(jsonBuilder()
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type1", mapping).mapperService();
+        MappedFieldType completionFieldType = mapperService.fullName("completion");
+        ParsedDocument parsedDocument = mapperService.documentMapper().parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(jsonBuilder()
                         .startObject()
                         .startArray("completion")
                         .startObject()
@@ -89,7 +86,7 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testIndexingWithSimpleContexts() throws Exception {
-        String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1")
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("completion")
                 .field("type", "completion")
                 .startArray("contexts")
@@ -100,13 +97,12 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endArray()
                 .endObject()
                 .endObject()
-                .endObject().endObject());
+                .endObject().endObject();
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(jsonBuilder()
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type1", mapping).mapperService();
+        MappedFieldType completionFieldType = mapperService.fullName("completion");
+        ParsedDocument parsedDocument = mapperService.documentMapper().parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(jsonBuilder()
                         .startObject()
                         .startArray("completion")
                         .startObject()
@@ -127,7 +123,7 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testIndexingWithContextList() throws Exception {
-        String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1")
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("completion")
                 .field("type", "completion")
                 .startArray("contexts")
@@ -137,13 +133,12 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endArray()
                 .endObject().endObject()
-                .endObject().endObject());
+                .endObject().endObject();
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(jsonBuilder()
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type1", mapping).mapperService();
+        MappedFieldType completionFieldType = mapperService.fullName("completion");
+        ParsedDocument parsedDocument = mapperService.documentMapper().parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(jsonBuilder()
                         .startObject()
                             .startObject("completion")
                                 .array("input", "suggestion5", "suggestion6", "suggestion7")
@@ -168,7 +163,7 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testIndexingWithMultipleContexts() throws Exception {
-        String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1")
+        XContentBuilder mapping = jsonBuilder().startObject().startObject("type1")
                 .startObject("properties").startObject("completion")
                 .field("type", "completion")
                 .startArray("contexts")
@@ -182,11 +177,10 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endArray()
                 .endObject().endObject()
-                .endObject().endObject());
+                .endObject().endObject();
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
-        FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion");
-        MappedFieldType completionFieldType = fieldMapper.fieldType();
+        MapperService mapperService = createIndex("test", Settings.EMPTY, "type1", mapping).mapperService();
+        MappedFieldType completionFieldType = mapperService.fullName("completion");
         XContentBuilder builder = jsonBuilder()
                 .startObject()
                 .startArray("completion")
@@ -200,8 +194,8 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endArray()
                 .endObject();
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference.bytes(builder),
-                XContentType.JSON));
+        ParsedDocument parsedDocument = mapperService.documentMapper().parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(builder), XContentType.JSON));
         IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
         assertContextSuggestFields(fields, 3);
     }
diff --git a/server/src/test/resources/org/elasticsearch/search/query/all-query-index.json b/server/src/test/resources/org/elasticsearch/search/query/all-query-index.json
index 72c9b54f6e3..abdc1192822 100644
--- a/server/src/test/resources/org/elasticsearch/search/query/all-query-index.json
+++ b/server/src/test/resources/org/elasticsearch/search/query/all-query-index.json
@@ -11,6 +11,10 @@
         "f1": {"type": "text"},
         "f2": {"type": "keyword"},
         "f3": {"type": "text"},
+        "f3_alias": {
+          "type": "alias",
+          "path": "f3"
+        },
         "f4": {
           "type": "text",
           "index_options": "docs"
@@ -42,6 +46,10 @@
           "format": "yyyy/MM/dd||epoch_millis"
         },
         "f_bool": {"type": "boolean"},
+        "f_bool_alias": {
+          "type": "alias",
+          "path": "f_bool"
+        },
         "f_byte": {"type": "byte"},
         "f_short": {"type": "short"},
         "f_int": {"type": "integer"},
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index e84f2a99a11..3002711bdbd 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -62,6 +62,7 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.mock.orig.Mockito;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
@@ -69,7 +70,6 @@ import org.elasticsearch.search.fetch.subphase.FetchSourceSubPhase;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalAggregationTestCase;
 import org.junit.After;
@@ -79,7 +79,12 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 import static org.mockito.Matchers.anyObject;
@@ -144,16 +149,50 @@ public abstract class AggregatorTestCase extends ESTestCase {
         SearchLookup searchLookup = new SearchLookup(mapperService, ifds::getForField, new String[]{TYPE_NAME});
         when(searchContext.lookup()).thenReturn(searchLookup);
 
-        QueryShardContext queryShardContext = queryShardContextMock(mapperService, fieldTypes, circuitBreakerService);
+        QueryShardContext queryShardContext = queryShardContextMock(mapperService);
         when(queryShardContext.getIndexSettings()).thenReturn(indexSettings);
         when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
-        for (MappedFieldType fieldType : fieldTypes) {
-            when(searchContext.smartNameFieldType(fieldType.name())).thenReturn(fieldType);
-        }
+
+        Map<String, MappedFieldType> fieldNameToType = new HashMap<>();
+        fieldNameToType.putAll(Arrays.stream(fieldTypes)
+            .collect(Collectors.toMap(MappedFieldType::name, Function.identity())));
+        fieldNameToType.putAll(getFieldAliases(fieldTypes));
+
+        registerFieldTypes(queryShardContext, searchContext, mapperService,
+            circuitBreakerService, fieldNameToType);
 
         return aggregationBuilder.build(searchContext, null);
     }
 
+    /**
+     * Allows subclasses to provide alternate names for the provided field type, which
+     * can be useful when testing aggregations on field aliases.
+     */
+    protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {
+        return Collections.emptyMap();
+    }
+
+    private void registerFieldTypes(QueryShardContext queryShardContext,
+                                    SearchContext searchContext,
+                                    MapperService mapperService,
+                                    CircuitBreakerService circuitBreakerService,
+                                    Map<String, MappedFieldType> fieldNameToType) {
+        for (Map.Entry<String, MappedFieldType> entry : fieldNameToType.entrySet()) {
+            String fieldName = entry.getKey();
+            MappedFieldType fieldType = entry.getValue();
+
+            when(queryShardContext.fieldMapper(fieldName)).thenReturn(fieldType);
+            when(searchContext.smartNameFieldType(fieldName)).thenReturn(fieldType);
+        }
+
+        for (MappedFieldType fieldType : new HashSet<>(fieldNameToType.values())) {
+            when(queryShardContext.getForField(fieldType)).then(invocation ->
+                fieldType.fielddataBuilder(mapperService.getIndexSettings().getIndex().getName())
+                    .build(mapperService.getIndexSettings(), fieldType,
+                        new IndexFieldDataCache.None(), circuitBreakerService, mapperService));
+        }
+    }
+
     protected <A extends Aggregator> A createAggregator(AggregationBuilder aggregationBuilder,
                                                         IndexSearcher indexSearcher,
                                                         MappedFieldType... fieldTypes) throws IOException {
@@ -257,16 +296,9 @@ public abstract class AggregatorTestCase extends ESTestCase {
     /**
      * sub-tests that need a more complex mock can overwrite this
      */
-    protected QueryShardContext queryShardContextMock(MapperService mapperService, MappedFieldType[] fieldTypes,
-            CircuitBreakerService circuitBreakerService) {
+    protected QueryShardContext queryShardContextMock(MapperService mapperService) {
         QueryShardContext queryShardContext = mock(QueryShardContext.class);
         when(queryShardContext.getMapperService()).thenReturn(mapperService);
-        for (MappedFieldType fieldType : fieldTypes) {
-            when(queryShardContext.fieldMapper(fieldType.name())).thenReturn(fieldType);
-            when(queryShardContext.getForField(fieldType)).then(invocation -> fieldType.fielddataBuilder(mapperService.getIndexSettings()
-                .getIndex().getName())
-                .build(mapperService.getIndexSettings(), fieldType, new IndexFieldDataCache.None(), circuitBreakerService, mapperService));
-        }
         NestedScope nestedScope = new NestedScope();
         when(queryShardContext.isFilter()).thenCallRealMethod();
         Mockito.doCallRealMethod().when(queryShardContext).setIsFilter(Matchers.anyBoolean());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index e50e93ec672..a891f30b93d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -83,7 +83,9 @@ import java.lang.reflect.Proxy;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -94,21 +96,35 @@ import static java.util.stream.Collectors.toList;
 public abstract class AbstractBuilderTestCase extends ESTestCase {
 
     public static final String STRING_FIELD_NAME = "mapped_string";
+    public static final String STRING_ALIAS_FIELD_NAME = "mapped_string_alias";
     protected static final String STRING_FIELD_NAME_2 = "mapped_string_2";
     protected static final String INT_FIELD_NAME = "mapped_int";
+    protected static final String INT_ALIAS_FIELD_NAME = "mapped_int_field_alias";
     protected static final String INT_RANGE_FIELD_NAME = "mapped_int_range";
     protected static final String DOUBLE_FIELD_NAME = "mapped_double";
     protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
     protected static final String DATE_FIELD_NAME = "mapped_date";
+    protected static final String DATE_ALIAS_FIELD_NAME = "mapped_date_alias";
     protected static final String DATE_RANGE_FIELD_NAME = "mapped_date_range";
     protected static final String OBJECT_FIELD_NAME = "mapped_object";
     protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point";
+    protected static final String GEO_POINT_ALIAS_FIELD_NAME = "mapped_geo_point_alias";
     protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape";
-    protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME, INT_RANGE_FIELD_NAME,
-            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME,
-            GEO_SHAPE_FIELD_NAME};
-    protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME, INT_RANGE_FIELD_NAME,
-            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME,  GEO_POINT_FIELD_NAME, };
+    protected static final String[] MAPPED_FIELD_NAMES = new String[]{STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME,
+        INT_FIELD_NAME, INT_RANGE_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME,
+        DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME,
+        GEO_SHAPE_FIELD_NAME};
+    protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[]{STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME,
+        INT_FIELD_NAME, INT_RANGE_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME,
+        DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME,  GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME};
+
+    private static final Map<String, String> ALIAS_TO_CONCRETE_FIELD_NAME = new HashMap<>();
+    static {
+        ALIAS_TO_CONCRETE_FIELD_NAME.put(STRING_ALIAS_FIELD_NAME, STRING_FIELD_NAME);
+        ALIAS_TO_CONCRETE_FIELD_NAME.put(INT_ALIAS_FIELD_NAME, INT_FIELD_NAME);
+        ALIAS_TO_CONCRETE_FIELD_NAME.put(DATE_ALIAS_FIELD_NAME, DATE_FIELD_NAME);
+        ALIAS_TO_CONCRETE_FIELD_NAME.put(GEO_POINT_ALIAS_FIELD_NAME, GEO_POINT_FIELD_NAME);
+    }
 
     protected static Version indexVersionCreated;
 
@@ -200,6 +216,13 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
             .build();
     }
 
+    protected static String expectedFieldName(String builderFieldName) {
+        if (currentTypes.length == 0) {
+            return builderFieldName;
+        }
+        return ALIAS_TO_CONCRETE_FIELD_NAME.getOrDefault(builderFieldName, builderFieldName);
+    }
+
     @AfterClass
     public static void afterClass() throws Exception {
         IOUtils.close(serviceHolder);
@@ -356,19 +379,24 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
                 }
             });
 
+
             for (String type : currentTypes) {
                 mapperService.merge(type, new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef(type,
-                        STRING_FIELD_NAME, "type=text",
-                        STRING_FIELD_NAME_2, "type=keyword",
-                        INT_FIELD_NAME, "type=integer",
-                        INT_RANGE_FIELD_NAME, "type=integer_range",
-                        DOUBLE_FIELD_NAME, "type=double",
-                        BOOLEAN_FIELD_NAME, "type=boolean",
-                        DATE_FIELD_NAME, "type=date",
-                        DATE_RANGE_FIELD_NAME, "type=date_range",
-                        OBJECT_FIELD_NAME, "type=object",
-                        GEO_POINT_FIELD_NAME, "type=geo_point",
-                        GEO_SHAPE_FIELD_NAME, "type=geo_shape"
+                    STRING_FIELD_NAME, "type=text",
+                    STRING_FIELD_NAME_2, "type=keyword",
+                    STRING_ALIAS_FIELD_NAME, "type=alias,path=" + STRING_FIELD_NAME,
+                    INT_FIELD_NAME, "type=integer",
+                    INT_ALIAS_FIELD_NAME, "type=alias,path=" + INT_FIELD_NAME,
+                    INT_RANGE_FIELD_NAME, "type=integer_range",
+                    DOUBLE_FIELD_NAME, "type=double",
+                    BOOLEAN_FIELD_NAME, "type=boolean",
+                    DATE_FIELD_NAME, "type=date",
+                    DATE_ALIAS_FIELD_NAME, "type=alias,path=" + DATE_FIELD_NAME,
+                    DATE_RANGE_FIELD_NAME, "type=date_range",
+                    OBJECT_FIELD_NAME, "type=object",
+                    GEO_POINT_FIELD_NAME, "type=geo_point",
+                    GEO_POINT_ALIAS_FIELD_NAME, "type=alias,path=" + GEO_POINT_FIELD_NAME,
+                    GEO_SHAPE_FIELD_NAME, "type=geo_shape"
                 ))), MapperService.MergeReason.MAPPING_UPDATE);
                 // also add mappings for two inner field in the object field
                 mapperService.merge(type, new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\","
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index a2acc5371a1..c1efd9d8e6a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -522,7 +522,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
      */
     protected abstract void doAssertLuceneQuery(QB queryBuilder, Query query, SearchContext context) throws IOException;
 
-    protected static void assertTermOrBoostQuery(Query query, String field, String value, float fieldBoost) {
+    protected void assertTermOrBoostQuery(Query query, String field, String value, float fieldBoost) {
         if (fieldBoost != AbstractQueryBuilder.DEFAULT_BOOST) {
             assertThat(query, instanceOf(BoostQuery.class));
             BoostQuery boostQuery = (BoostQuery) query;
@@ -532,10 +532,12 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         assertTermQuery(query, field, value);
     }
 
-    protected static void assertTermQuery(Query query, String field, String value) {
+    protected void assertTermQuery(Query query, String field, String value) {
         assertThat(query, instanceOf(TermQuery.class));
         TermQuery termQuery = (TermQuery) query;
-        assertThat(termQuery.getTerm().field(), equalTo(field));
+
+        String expectedFieldName = expectedFieldName(field);
+        assertThat(termQuery.getTerm().field(), equalTo(expectedFieldName));
         assertThat(termQuery.getTerm().text().toLowerCase(Locale.ROOT), equalTo(value.toLowerCase(Locale.ROOT)));
     }
 
@@ -625,6 +627,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         Object value;
         switch (fieldName) {
             case STRING_FIELD_NAME:
+            case STRING_ALIAS_FIELD_NAME:
                 if (rarely()) {
                     // unicode in 10% cases
                     JsonStringEncoder encoder = JsonStringEncoder.getInstance();
@@ -783,4 +786,8 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         Rewriteable.rewriteAndFetch(builder, context, future);
         return future.actionGet();
     }
+
+    public boolean isTextField(String fieldName) {
+        return fieldName.equals(STRING_FIELD_NAME) || fieldName.equals(STRING_ALIAS_FIELD_NAME);
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
index d7d8c2ceb1b..dd41bc5a7fd 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
@@ -30,8 +30,10 @@ import org.elasticsearch.indices.IndicesRequestCache;
 import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -161,10 +163,12 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
                 .build();
     }
 
-    public void testQuery() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("test")
-                        .addMapping("type1", "field1", "type=text", "field2", "type=text", "field3", "type=text")
-        );
+    public void testQuery() {
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("type1",
+            "field1", "type=text",
+            "field2", "type=text",
+            "field3", "type=text",
+            "alias", "type=alias,path=field1"));
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2", "field3", "value3")
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
@@ -299,6 +303,20 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
                 .setQuery(matchQuery("field3", "value3"))
                 .get();
         assertHitCount(response, 0);
+
+        // user1 has access to field1, so a query on its field alias should match with the document:
+        response = client()
+                .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                .prepareSearch("test")
+                .setQuery(matchQuery("alias", "value1"))
+                .get();
+        assertHitCount(response, 1);
+        // user2 has no access to field1, so a query on its field alias should not match with the document:
+        response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                .prepareSearch("test")
+                .setQuery(matchQuery("alias", "value1"))
+                .get();
+        assertHitCount(response, 0);
     }
 
     public void testGetApi() throws Exception {
@@ -793,10 +811,11 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
     }
 
     public void testFields() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("test")
-                        .addMapping("type1", "field1", "type=text,store=true", "field2", "type=text,store=true",
-                                "field3", "type=text,store=true")
-        );
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("type1",
+            "field1", "type=text,store=true",
+            "field2", "type=text,store=true",
+            "field3", "type=text,store=true",
+            "alias", "type=alias,path=field1"));
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2", "field3", "value3")
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
@@ -888,6 +907,22 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
         assertThat(response.getHits().getAt(0).getFields().size(), equalTo(2));
         assertThat(response.getHits().getAt(0).getFields().get("field1").<String>getValue(), equalTo("value1"));
         assertThat(response.getHits().getAt(0).getFields().get("field2").<String>getValue(), equalTo("value2"));
+
+        // user1 is granted access to field1 only, and so should be able to load it by alias:
+        response = client()
+                .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                .prepareSearch("test")
+                .addStoredField("alias")
+                .get();
+        assertThat(response.getHits().getAt(0).getFields().size(), equalTo(1));
+        assertThat(response.getHits().getAt(0).getFields().get("alias").getValue(), equalTo("value1"));
+
+        // user2 is not granted access to field1, and so should not be able to load it by alias:
+        response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                .prepareSearch("test")
+                .addStoredField("alias")
+                .get();
+        assertThat(response.getHits().getAt(0).getFields().size(), equalTo(0));
     }
 
     public void testSource() throws Exception {
@@ -963,11 +998,11 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
         assertThat(response.getHits().getAt(0).getSourceAsMap().get("field2").toString(), equalTo("value2"));
     }
 
-    public void testSort() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("test")
-                        .addMapping("type1", "field1", "type=long", "field2", "type=long")
-        );
-
+    public void testSort() {
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("type1",
+            "field1", "type=long",
+            "field2", "type=long",
+            "alias", "type=alias,path=field1"));
         client().prepareIndex("test", "type1", "1").setSource("field1", 1d, "field2", 2d)
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
@@ -1000,12 +1035,81 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
                 .addSort("field2", SortOrder.ASC)
                 .get();
         assertThat(response.getHits().getAt(0).getSortValues()[0], equalTo(2L));
+
+        // user1 is granted to use field1, so it is included in the sort_values when using its alias:
+        response = client()
+                .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                .prepareSearch("test")
+                .addSort("alias", SortOrder.ASC)
+                .get();
+        assertThat(response.getHits().getAt(0).getSortValues()[0], equalTo(1L));
+
+        // user2 is not granted to use field1, so the default missing sort value is included when using its alias:
+        response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                .prepareSearch("test")
+                .addSort("alias", SortOrder.ASC)
+                .get();
+        assertThat(response.getHits().getAt(0).getSortValues()[0], equalTo(Long.MAX_VALUE));
     }
 
-    public void testAggs() throws Exception {
-        assertAcked(client().admin().indices().prepareCreate("test")
-                        .addMapping("type1", "field1", "type=text,fielddata=true", "field2", "type=text,fielddata=true")
-        );
+     public void testHighlighting() {
+         assertAcked(client().admin().indices().prepareCreate("test").addMapping("type1",
+             "field1", "type=text",
+             "field2", "type=text",
+             "field3", "type=text",
+             "alias", "type=alias,path=field1"));
+         client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2", "field3", "value3")
+             .setRefreshPolicy(IMMEDIATE)
+             .get();
+
+         // user1 has access to field1, so the highlight should be visible:
+         SearchResponse response = client()
+             .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+             .prepareSearch("test")
+             .setQuery(matchQuery("field1", "value1"))
+             .highlighter(new HighlightBuilder().field("field1"))
+             .get();
+         assertHitCount(response, 1);
+         SearchHit hit = response.getHits().iterator().next();
+         assertEquals(hit.getHighlightFields().size(), 1);
+
+         // user2 has no access to field1, so the highlight should not be visible:
+         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+             .prepareSearch("test")
+             .setQuery(matchQuery("field2", "value2"))
+             .highlighter(new HighlightBuilder().field("field1"))
+             .get();
+         assertHitCount(response, 1);
+         hit = response.getHits().iterator().next();
+         assertEquals(hit.getHighlightFields().size(), 0);
+
+         // user1 has access to field1, so the highlight on its alias should be visible:
+         response = client()
+             .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+             .prepareSearch("test")
+             .setQuery(matchQuery("field1", "value1"))
+             .highlighter(new HighlightBuilder().field("alias"))
+             .get();
+         assertHitCount(response, 1);
+         hit = response.getHits().iterator().next();
+         assertEquals(hit.getHighlightFields().size(), 1);
+
+         // user2 has no access to field1, so the highlight on its alias should not be visible:
+         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+             .prepareSearch("test")
+             .setQuery(matchQuery("field2", "value2"))
+             .highlighter(new HighlightBuilder().field("alias"))
+             .get();
+         assertHitCount(response, 1);
+         hit = response.getHits().iterator().next();
+         assertEquals(hit.getHighlightFields().size(), 0);
+     }
+
+    public void testAggs() {
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("type1",
+            "field1", "type=text,fielddata=true",
+            "field2", "type=text,fielddata=true",
+            "alias", "type=alias,path=field1"));
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2")
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
@@ -1038,6 +1142,21 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
                 .addAggregation(AggregationBuilders.terms("_name").field("field2"))
                 .get();
         assertThat(((Terms) response.getAggregations().get("_name")).getBucketByKey("value2").getDocCount(), equalTo(1L));
+
+        // user1 is authorized to use field1, so buckets are include for a term agg on its alias:
+        response = client()
+                .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                .prepareSearch("test")
+                .addAggregation(AggregationBuilders.terms("_name").field("alias"))
+                .get();
+        assertThat(((Terms) response.getAggregations().get("_name")).getBucketByKey("value1").getDocCount(), equalTo(1L));
+
+        // user2 is not authorized to use field1, so no buckets are include for a term agg on its alias:
+        response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                .prepareSearch("test")
+                .addAggregation(AggregationBuilders.terms("_name").field("alias"))
+                .get();
+        assertThat(((Terms) response.getAggregations().get("_name")).getBucketByKey("value1"), nullValue());
     }
 
     public void testTVApi() throws Exception {
@@ -1218,15 +1337,22 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
 
     public void testParentChild() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
-              .startObject("properties")
-                  .startObject("join_field")
-                      .field("type", "join")
-                      .startObject("relations")
-                          .field("parent", "child")
-                      .endObject()
-                  .endObject()
-              .endObject()
-              .endObject();
+            .startObject("properties")
+                .startObject("field1")
+                    .field("type", "keyword")
+                .endObject()
+                .startObject("alias")
+                    .field("type", "alias")
+                    .field("path", "field1")
+                .endObject()
+                .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                        .field("parent", "child")
+                    .endObject()
+                 .endObject()
+            .endObject()
+            .endObject();
         assertAcked(prepareCreate("test")
               .addMapping("doc", mapping));
         ensureGreen();
@@ -1264,6 +1390,23 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
                 .setQuery(hasChildQuery("child", termQuery("field1", "yellow"), ScoreMode.None))
                 .get();
         assertHitCount(searchResponse, 0L);
+
+        // Perform the same checks, but using an alias for field1.
+        searchResponse = client()
+            .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+            .prepareSearch("test")
+            .setQuery(hasChildQuery("child", termQuery("alias", "yellow"), ScoreMode.None))
+            .get();
+        assertHitCount(searchResponse, 1L);
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
+        assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("p1"));
+
+        searchResponse = client()
+            .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+            .prepareSearch("test")
+            .setQuery(hasChildQuery("child", termQuery("alias", "yellow"), ScoreMode.None))
+            .get();
+        assertHitCount(searchResponse, 0L);
     }
 
     public void testUpdateApiIsBlocked() throws Exception {
@@ -1315,10 +1458,9 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
         assertThat(client().prepareGet("test", "type", "1").get().getSource().get("field2").toString(), equalTo("value3"));
     }
 
-    public void testQuery_withRoleWithFieldWildcards() throws Exception {
+    public void testQuery_withRoleWithFieldWildcards() {
         assertAcked(client().admin().indices().prepareCreate("test")
-                        .addMapping("type1", "field1", "type=text", "field2", "type=text")
-        );
+                        .addMapping("type1", "field1", "type=text", "field2", "type=text"));
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2")
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
@@ -1345,9 +1487,12 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
     }
 
     public void testExistQuery() {
-        assertAcked(client().admin().indices().prepareCreate("test")
-                .addMapping("type1", "field1", "type=text", "field2", "type=text", "field3", "type=text")
-        );
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("type1",
+            "field1", "type=text",
+            "field2", "type=text",
+            "field3", "type=text",
+            "alias", "type=alias,path=field1"));
+
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2", "field3", "value3")
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
@@ -1402,6 +1547,20 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
                 .setQuery(existsQuery("field2"))
                 .get();
         assertHitCount(response, 0);
+
+        // user1 has access to field1, so a query on its alias should match with the document:
+        response = client()
+                .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                .prepareSearch("test")
+                .setQuery(existsQuery("alias"))
+                .get();
+        assertHitCount(response, 1);
+        // user2 has no access to field1, so the query should not match with the document:
+        response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                .prepareSearch("test")
+                .setQuery(existsQuery("alias"))
+                .get();
+        assertHitCount(response, 0);
     }
 
 }

From e20f59aa716f094296960afdc39f927a987481b5 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Wed, 18 Jul 2018 10:26:26 -0700
Subject: [PATCH 089/260] [test] use randomized runner in packaging tests
 (#32109)

Use the randomized runner from the test framework and add some basic
logging to make the packaging tests behave more similarly to how we use
junit in the rest of the project
---
 qa/vagrant/build.gradle                       | 18 +++++--
 .../packaging/test/ArchiveTestCase.java       | 16 ++----
 .../packaging/test/PackagingTestCase.java     | 51 +++++++++++++++++++
 3 files changed, 67 insertions(+), 18 deletions(-)
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackagingTestCase.java

diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 704136eb4cf..37190632b44 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -28,8 +28,8 @@ plugins {
 
 dependencies {
   compile "junit:junit:${versions.junit}"
-  compile "org.hamcrest:hamcrest-core:${versions.hamcrest}"
-  compile "org.hamcrest:hamcrest-library:${versions.hamcrest}"
+  compile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+  compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
 
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
@@ -81,7 +81,7 @@ tasks.dependencyLicenses.enabled = false
 tasks.dependenciesInfo.enabled = false
 
 tasks.thirdPartyAudit.excludes = [
-        //commons-logging optional dependencies
+        // commons-logging optional dependencies
         'org.apache.avalon.framework.logger.Logger',
         'org.apache.log.Hierarchy',
         'org.apache.log.Logger',
@@ -89,7 +89,15 @@ tasks.thirdPartyAudit.excludes = [
         'org.apache.log4j.Level',
         'org.apache.log4j.Logger',
         'org.apache.log4j.Priority',
-        //commons-logging provided dependencies
+        // commons-logging provided dependencies
         'javax.servlet.ServletContextEvent',
-        'javax.servlet.ServletContextListener'
+        'javax.servlet.ServletContextListener',
+        // from randomized testing
+        'org.apache.tools.ant.BuildException',
+        'org.apache.tools.ant.DirectoryScanner',
+        'org.apache.tools.ant.Task',
+        'org.apache.tools.ant.types.FileSet',
+        'org.easymock.EasyMock',
+        'org.easymock.IArgumentMatcher',
+        'org.jmock.core.Constraint'
 ]
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
index df5e8cf995d..3aada7837d8 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.packaging.test;
 
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
 import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.util.Archives;
 import org.elasticsearch.packaging.util.Platforms;
@@ -27,9 +28,6 @@ import org.elasticsearch.packaging.util.Shell;
 import org.elasticsearch.packaging.util.Shell.Result;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.FixMethodOrder;
-import org.junit.Test;
-import org.junit.runners.MethodSorters;
 
 import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.Installation;
@@ -67,8 +65,8 @@ import static org.junit.Assume.assumeTrue;
  * Tests that apply to the archive distributions (tar, zip). To add a case for a distribution, subclass and
  * override {@link ArchiveTestCase#distribution()}. These tests should be the same across all archive distributions
  */
-@FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public abstract class ArchiveTestCase {
+@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
+public abstract class ArchiveTestCase extends PackagingTestCase {
 
     private static Installation installation;
 
@@ -86,13 +84,11 @@ public abstract class ArchiveTestCase {
         assumeTrue("only compatible distributions", distribution().packaging.compatible);
     }
 
-    @Test
     public void test10Install() {
         installation = installArchive(distribution());
         verifyArchiveInstallation(installation, distribution());
     }
 
-    @Test
     public void test20PluginsListWithNoPlugins() {
         assumeThat(installation, is(notNullValue()));
 
@@ -103,7 +99,6 @@ public abstract class ArchiveTestCase {
         assertThat(r.stdout, isEmptyString());
     }
 
-    @Test
     public void test30AbortWhenJavaMissing() {
         assumeThat(installation, is(notNullValue()));
 
@@ -146,7 +141,6 @@ public abstract class ArchiveTestCase {
         });
     }
 
-    @Test
     public void test40CreateKeystoreManually() {
         assumeThat(installation, is(notNullValue()));
 
@@ -180,7 +174,6 @@ public abstract class ArchiveTestCase {
         });
     }
 
-    @Test
     public void test50StartAndStop() throws IOException {
         assumeThat(installation, is(notNullValue()));
 
@@ -198,7 +191,6 @@ public abstract class ArchiveTestCase {
         Archives.stopElasticsearch(installation);
     }
 
-    @Test
     public void test60AutoCreateKeystore() {
         assumeThat(installation, is(notNullValue()));
 
@@ -218,7 +210,6 @@ public abstract class ArchiveTestCase {
         });
     }
 
-    @Test
     public void test70CustomPathConfAndJvmOptions() throws IOException {
         assumeThat(installation, is(notNullValue()));
 
@@ -268,7 +259,6 @@ public abstract class ArchiveTestCase {
         }
     }
 
-    @Test
     public void test80RelativePathConf() throws IOException {
         assumeThat(installation, is(notNullValue()));
 
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackagingTestCase.java
new file mode 100644
index 00000000000..77644b70f28
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackagingTestCase.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import com.carrotsearch.randomizedtesting.JUnit3MethodProvider;
+import com.carrotsearch.randomizedtesting.RandomizedRunner;
+import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+@RunWith(RandomizedRunner.class)
+@TestMethodProviders({
+    JUnit3MethodProvider.class
+})
+/**
+ * Class that all packaging test cases should inherit from. This makes working with the packaging tests more similar to what we're
+ * familiar with from {@link org.elasticsearch.test.ESTestCase} without having to apply its behavior that's not relevant here
+ */
+public abstract class PackagingTestCase {
+
+    protected final Log logger = LogFactory.getLog(getClass());
+
+    @Rule
+    public final TestName testNameRule = new TestName();
+
+    @Before
+    public void logTestNameBefore() {
+        logger.info("[" + testNameRule.getMethodName() + "]: before test");
+    }
+}

From 90fcb38448158d3a5d595307ee9f5c3d64b42e7f Mon Sep 17 00:00:00 2001
From: Tim Brooks <tim@uncontended.net>
Date: Wed, 18 Jul 2018 11:49:42 -0600
Subject: [PATCH 090/260] Adjust SSLDriver behavior for JDK11 changes (#32145)

This is related to #32122. A number of things changed related to adding
TLS 1.3 support in JDK11. Some exception messages and other SSLEngine
behavior changed. This commit fixes assertions on exception messages.
Additionally it identifies two bugs related to how the SSLDriver behaves
in regards to JDK11 changes. Finally, it mutes a tests until correct
behavior can be identified. There is another open issue for that muted
test (#32144).
---
 .../security/transport/nio/SSLDriver.java     | 19 ++++++--
 .../transport/nio/SSLDriverTests.java         | 43 +++++++++++++++----
 2 files changed, 49 insertions(+), 13 deletions(-)

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLDriver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLDriver.java
index 382230684c7..fa7791689aa 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLDriver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLDriver.java
@@ -349,7 +349,10 @@ public class SSLDriver implements AutoCloseable {
                         if (hasFlushPending() == false) {
                             handshakeStatus = wrap(EMPTY_BUFFER_ARRAY).getHandshakeStatus();
                         }
-                        continueHandshaking = false;
+                        // If we need NEED_TASK we should run the tasks immediately
+                        if (handshakeStatus != SSLEngineResult.HandshakeStatus.NEED_TASK) {
+                            continueHandshaking = false;
+                        }
                         break;
                     case NEED_TASK:
                         runTasks();
@@ -432,8 +435,16 @@ public class SSLDriver implements AutoCloseable {
         }
 
         private void maybeFinishHandshake() {
-            // We only acknowledge that we are done handshaking if there are no bytes that need to be written
-            if (hasFlushPending() == false) {
+            if (engine.isOutboundDone() || engine.isInboundDone()) {
+                // If the engine is partially closed, immediate transition to close mode.
+                if (currentMode.isHandshake()) {
+                    currentMode = new CloseMode(true);
+                } else {
+                    String message = "Expected to be in handshaking mode. Instead in non-handshaking mode: " + currentMode;
+                    throw new AssertionError(message);
+                }
+            } else if (hasFlushPending() == false) {
+                // We only acknowledge that we are done handshaking if there are no bytes that need to be written
                 if (currentMode.isHandshake()) {
                     currentMode = new ApplicationMode();
                 } else {
@@ -510,7 +521,7 @@ public class SSLDriver implements AutoCloseable {
             if (isHandshaking && engine.isInboundDone() == false) {
                 // If we attempt to close during a handshake either we are sending an alert and inbound
                 // should already be closed or we are sending a close_notify. If we send a close_notify
-                // the peer will send an handshake error alert. If we attempt to receive the handshake alert,
+                // the peer might send an handshake error alert. If we attempt to receive the handshake alert,
                 // the engine will throw an IllegalStateException as it is not in a proper state to receive
                 // handshake message. Closing inbound immediately after close_notify is the cleanest option.
                 needToReceiveClose = false;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLDriverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLDriverTests.java
index e1e05032014..303ed92130a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLDriverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLDriverTests.java
@@ -57,8 +57,15 @@ public class SSLDriverTests extends ESTestCase {
     public void testRenegotiate() throws Exception {
         SSLContext sslContext = getSSLContext();
 
-        SSLDriver clientDriver = getDriver(sslContext.createSSLEngine(), true);
-        SSLDriver serverDriver = getDriver(sslContext.createSSLEngine(), false);
+        SSLEngine serverEngine = sslContext.createSSLEngine();
+        SSLEngine clientEngine = sslContext.createSSLEngine();
+
+        String[] serverProtocols = {"TLSv1.2"};
+        serverEngine.setEnabledProtocols(serverProtocols);
+        String[] clientProtocols = {"TLSv1.2"};
+        clientEngine.setEnabledProtocols(clientProtocols);
+        SSLDriver clientDriver = getDriver(clientEngine, true);
+        SSLDriver serverDriver = getDriver(serverEngine, false);
 
         handshake(clientDriver, serverDriver);
 
@@ -119,16 +126,27 @@ public class SSLDriverTests extends ESTestCase {
         SSLContext sslContext = getSSLContext();
         SSLEngine clientEngine = sslContext.createSSLEngine();
         SSLEngine serverEngine = sslContext.createSSLEngine();
-        String[] serverProtocols = {"TLSv1.1", "TLSv1.2"};
+        String[] serverProtocols = {"TLSv1.2"};
         serverEngine.setEnabledProtocols(serverProtocols);
-        String[] clientProtocols = {"TLSv1"};
+        String[] clientProtocols = {"TLSv1.1"};
         clientEngine.setEnabledProtocols(clientProtocols);
         SSLDriver clientDriver = getDriver(clientEngine, true);
         SSLDriver serverDriver = getDriver(serverEngine, false);
 
         SSLException sslException = expectThrows(SSLException.class, () -> handshake(clientDriver, serverDriver));
-        assertEquals("Client requested protocol TLSv1 not enabled or not supported", sslException.getMessage());
-        failedCloseAlert(serverDriver, clientDriver);
+        String oldExpected = "Client requested protocol TLSv1.1 not enabled or not supported";
+        String jdk11Expected = "Received fatal alert: protocol_version";
+        boolean expectedMessage = oldExpected.equals(sslException.getMessage()) || jdk11Expected.equals(sslException.getMessage());
+        assertTrue("Unexpected exception message: " + sslException.getMessage(), expectedMessage);
+
+        // In JDK11 we need an non-application write
+        if (serverDriver.needsNonApplicationWrite()) {
+            serverDriver.nonApplicationWrite();
+        }
+        // Prior to JDK11 we still need to send a close alert
+        if (serverDriver.isClosed() == false) {
+            failedCloseAlert(serverDriver, clientDriver);
+        }
     }
 
     public void testHandshakeFailureBecauseNoCiphers() throws Exception {
@@ -144,11 +162,18 @@ public class SSLDriverTests extends ESTestCase {
         SSLDriver clientDriver = getDriver(clientEngine, true);
         SSLDriver serverDriver = getDriver(serverEngine, false);
 
-        SSLException sslException = expectThrows(SSLException.class, () -> handshake(clientDriver, serverDriver));
-        assertEquals("no cipher suites in common", sslException.getMessage());
-        failedCloseAlert(serverDriver, clientDriver);
+        expectThrows(SSLException.class, () -> handshake(clientDriver, serverDriver));
+        // In JDK11 we need an non-application write
+        if (serverDriver.needsNonApplicationWrite()) {
+            serverDriver.nonApplicationWrite();
+        }
+        // Prior to JDK11 we still need to send a close alert
+        if (serverDriver.isClosed() == false) {
+            failedCloseAlert(serverDriver, clientDriver);
+        }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32144")
     public void testCloseDuringHandshake() throws Exception {
         SSLContext sslContext = getSSLContext();
         SSLDriver clientDriver = getDriver(sslContext.createSSLEngine(), true);

From 63749498ea92dc03ac424f7f5764eed752eb567e Mon Sep 17 00:00:00 2001
From: Michael Basnight <mbasnight@gmail.com>
Date: Wed, 18 Jul 2018 13:28:25 -0500
Subject: [PATCH 091/260] Remove empty @return from JavaDoc

---
 .../xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
index d35e7ab0b0c..d6b2e321d6c 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
@@ -318,8 +318,6 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
 
     /**
      * Collects all the certificates that are normally trusted by the node ( contained in testnode.jks )
-     *
-     * @return
      */
     List<String> getNodeTrustedCertificates() {
         Path testnodeCert =

From 27d663b64b775355bb787355888132837f28db2e Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Wed, 18 Jul 2018 16:44:05 -0700
Subject: [PATCH 092/260] Revert "Introduce a Hashing Processor (#31087)"
 (#32178)

This reverts commit 8c78fe711486a3f722879d8f7cf89632daf793e0.
---
 x-pack/plugin/build.gradle                    |   1 -
 .../xpack/security/Security.java              |  10 +-
 .../xpack/security/ingest/HashProcessor.java  | 200 ------------------
 .../ingest/HashProcessorFactoryTests.java     | 136 ------------
 .../security/ingest/HashProcessorTests.java   | 130 ------------
 .../test/hash_processor/10_basic.yml          |  51 -----
 6 files changed, 1 insertion(+), 527 deletions(-)
 delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java
 delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java
 delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java
 delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml

diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index 20ae41f10dc..ca529496bf1 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -151,7 +151,6 @@ integTestCluster {
   setting 'xpack.license.self_generated.type', 'trial'
   keystoreSetting 'bootstrap.password', 'x-pack-test-password'
   keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
-  keystoreSetting 'xpack.security.ingest.hash.processor.key', 'hmackey'
   distribution = 'zip' // this is important since we use the reindex module in ML
 
   setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 3115c08a946..98b5df3edc5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -175,7 +175,6 @@ import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
-import org.elasticsearch.xpack.security.ingest.HashProcessor;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
 import org.elasticsearch.xpack.security.rest.SecurityRestFilter;
 import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction;
@@ -580,10 +579,6 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         // hide settings
         settingsList.add(Setting.listSetting(SecurityField.setting("hide_settings"), Collections.emptyList(), Function.identity(),
                 Property.NodeScope, Property.Filtered));
-
-        // ingest processor settings
-        settingsList.add(HashProcessor.HMAC_KEY_SETTING);
-
         return settingsList;
     }
 
@@ -727,10 +722,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-        Map<String, Processor.Factory> processors = new HashMap<>();
-        processors.put(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
-        processors.put(HashProcessor.TYPE, new HashProcessor.Factory(parameters.env.settings()));
-        return processors;
+        return Collections.singletonMap(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
     }
 
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java
deleted file mode 100644
index fa49b843847..00000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.ingest;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.SecureSetting;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.ConfigurationUtils;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.xpack.core.security.SecurityField;
-
-import javax.crypto.Mac;
-import javax.crypto.SecretKeyFactory;
-import javax.crypto.spec.PBEKeySpec;
-import javax.crypto.spec.SecretKeySpec;
-import java.nio.charset.StandardCharsets;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import java.security.spec.InvalidKeySpecException;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
-
-/**
- * A processor that hashes the contents of a field (or fields) using various hashing algorithms
- */
-public final class HashProcessor extends AbstractProcessor {
-    public static final String TYPE = "hash";
-    public static final Setting.AffixSetting<SecureString> HMAC_KEY_SETTING = SecureSetting
-        .affixKeySetting(SecurityField.setting("ingest." + TYPE) + ".", "key",
-            (key) -> SecureSetting.secureString(key, null));
-
-    private final List<String> fields;
-    private final String targetField;
-    private final Method method;
-    private final Mac mac;
-    private final byte[] salt;
-    private final boolean ignoreMissing;
-
-    HashProcessor(String tag, List<String> fields, String targetField, byte[] salt, Method method, @Nullable Mac mac,
-                  boolean ignoreMissing) {
-        super(tag);
-        this.fields = fields;
-        this.targetField = targetField;
-        this.method = method;
-        this.mac = mac;
-        this.salt = salt;
-        this.ignoreMissing = ignoreMissing;
-    }
-
-    List<String> getFields() {
-        return fields;
-    }
-
-    String getTargetField() {
-        return targetField;
-    }
-
-    byte[] getSalt() {
-        return salt;
-    }
-
-    @Override
-    public void execute(IngestDocument document) {
-        Map<String, String> hashedFieldValues = fields.stream().map(f -> {
-            String value = document.getFieldValue(f, String.class, ignoreMissing);
-            if (value == null && ignoreMissing) {
-                return new Tuple<String, String>(null, null);
-            }
-            try {
-                return new Tuple<>(f, method.hash(mac, salt, value));
-            } catch (Exception e) {
-                throw new IllegalArgumentException("field[" + f + "] could not be hashed", e);
-            }
-        }).filter(tuple -> Objects.nonNull(tuple.v1())).collect(Collectors.toMap(Tuple::v1, Tuple::v2));
-        if (fields.size() == 1) {
-            document.setFieldValue(targetField, hashedFieldValues.values().iterator().next());
-        } else {
-            document.setFieldValue(targetField, hashedFieldValues);
-        }
-    }
-
-    @Override
-    public String getType() {
-        return TYPE;
-    }
-
-    public static final class Factory implements Processor.Factory {
-
-        private final Settings settings;
-        private final Map<String, SecureString> secureKeys;
-
-        public Factory(Settings settings) {
-            this.settings = settings;
-            this.secureKeys = new HashMap<>();
-            HMAC_KEY_SETTING.getAllConcreteSettings(settings).forEach(k -> {
-                secureKeys.put(k.getKey(), k.get(settings));
-            });
-        }
-
-        private static Mac createMac(Method method, SecureString password, byte[] salt, int iterations) {
-            try {
-                SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2With" + method.getAlgorithm());
-                PBEKeySpec keySpec = new PBEKeySpec(password.getChars(), salt, iterations, 128);
-                byte[] pbkdf2 = secretKeyFactory.generateSecret(keySpec).getEncoded();
-                Mac mac = Mac.getInstance(method.getAlgorithm());
-                mac.init(new SecretKeySpec(pbkdf2, method.getAlgorithm()));
-                return mac;
-            } catch (NoSuchAlgorithmException | InvalidKeySpecException | InvalidKeyException e) {
-                throw new IllegalArgumentException("invalid settings", e);
-            }
-        }
-
-        @Override
-        public HashProcessor create(Map<String, Processor.Factory> registry, String processorTag, Map<String, Object> config) {
-            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
-            List<String> fields = ConfigurationUtils.readList(TYPE, processorTag, config, "fields");
-            if (fields.isEmpty()) {
-                throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields", "must specify at least one field");
-            } else if (fields.stream().anyMatch(Strings::isNullOrEmpty)) {
-                throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields",
-                    "a field-name entry is either empty or null");
-            }
-            String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
-            String keySettingName = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "key_setting");
-            SecureString key = secureKeys.get(keySettingName);
-            if (key == null) {
-                throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "key_setting",
-                    "key [" + keySettingName + "] must match [xpack.security.ingest.hash.*.key]. It is not set");
-            }
-            String saltString = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "salt");
-            byte[] salt = saltString.getBytes(StandardCharsets.UTF_8);
-            String methodProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "method", "SHA256");
-            Method method = Method.fromString(processorTag, "method", methodProperty);
-            int iterations = ConfigurationUtils.readIntProperty(TYPE, processorTag, config, "iterations", 5);
-            Mac mac = createMac(method, key, salt, iterations);
-            return new HashProcessor(processorTag, fields, targetField, salt, method, mac, ignoreMissing);
-        }
-    }
-
-    enum Method {
-        SHA1("HmacSHA1"),
-        SHA256("HmacSHA256"),
-        SHA384("HmacSHA384"),
-        SHA512("HmacSHA512");
-
-        private final String algorithm;
-
-        Method(String algorithm) {
-            this.algorithm = algorithm;
-        }
-
-        public String getAlgorithm() {
-            return algorithm;
-        }
-
-        @Override
-        public String toString() {
-            return name().toLowerCase(Locale.ROOT);
-        }
-
-        public String hash(Mac mac, byte[] salt, String input) {
-            try {
-                byte[] encrypted = mac.doFinal(input.getBytes(StandardCharsets.UTF_8));
-                byte[] messageWithSalt = new byte[salt.length + encrypted.length];
-                System.arraycopy(salt, 0, messageWithSalt, 0, salt.length);
-                System.arraycopy(encrypted, 0, messageWithSalt, salt.length, encrypted.length);
-                return Base64.getEncoder().encodeToString(messageWithSalt);
-            } catch (IllegalStateException e) {
-                throw new ElasticsearchException("error hashing data", e);
-            }
-        }
-
-        public static Method fromString(String processorTag, String propertyName, String type) {
-            try {
-                return Method.valueOf(type.toUpperCase(Locale.ROOT));
-            } catch(IllegalArgumentException e) {
-                throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type +
-                    "] not supported, cannot convert field. Valid hash methods: " + Arrays.toString(Method.values()));
-            }
-        }
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java
deleted file mode 100644
index e9dda488e72..00000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.ingest;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.settings.MockSecureSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESTestCase;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class HashProcessorFactoryTests extends ESTestCase {
-
-    public void testProcessor() {
-        MockSecureSettings mockSecureSettings = new MockSecureSettings();
-        mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
-        Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
-        HashProcessor.Factory factory = new HashProcessor.Factory(settings);
-        Map<String, Object> config = new HashMap<>();
-        config.put("fields", Collections.singletonList("_field"));
-        config.put("target_field", "_target");
-        config.put("salt", "_salt");
-        config.put("key_setting", "xpack.security.ingest.hash.processor.key");
-        for (HashProcessor.Method method : HashProcessor.Method.values()) {
-            config.put("method", method.toString());
-            HashProcessor processor = factory.create(null, "_tag", new HashMap<>(config));
-            assertThat(processor.getFields(), equalTo(Collections.singletonList("_field")));
-            assertThat(processor.getTargetField(), equalTo("_target"));
-            assertArrayEquals(processor.getSalt(), "_salt".getBytes(StandardCharsets.UTF_8));
-        }
-    }
-
-    public void testProcessorNoFields() {
-        MockSecureSettings mockSecureSettings = new MockSecureSettings();
-        mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
-        Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
-        HashProcessor.Factory factory = new HashProcessor.Factory(settings);
-        Map<String, Object> config = new HashMap<>();
-        config.put("target_field", "_target");
-        config.put("salt", "_salt");
-        config.put("key_setting", "xpack.security.ingest.hash.processor.key");
-        config.put("method", HashProcessor.Method.SHA1.toString());
-        ElasticsearchException e = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, "_tag", config));
-        assertThat(e.getMessage(), equalTo("[fields] required property is missing"));
-    }
-
-    public void testProcessorNoTargetField() {
-        MockSecureSettings mockSecureSettings = new MockSecureSettings();
-        mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
-        Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
-        HashProcessor.Factory factory = new HashProcessor.Factory(settings);
-        Map<String, Object> config = new HashMap<>();
-        config.put("fields", Collections.singletonList("_field"));
-        config.put("salt", "_salt");
-        config.put("key_setting", "xpack.security.ingest.hash.processor.key");
-        config.put("method", HashProcessor.Method.SHA1.toString());
-        ElasticsearchException e = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, "_tag", config));
-        assertThat(e.getMessage(), equalTo("[target_field] required property is missing"));
-    }
-
-    public void testProcessorFieldsIsEmpty() {
-        MockSecureSettings mockSecureSettings = new MockSecureSettings();
-        mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
-        Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
-        HashProcessor.Factory factory = new HashProcessor.Factory(settings);
-        Map<String, Object> config = new HashMap<>();
-        config.put("fields", Collections.singletonList(randomBoolean() ? "" : null));
-        config.put("salt", "_salt");
-        config.put("target_field", "_target");
-        config.put("key_setting", "xpack.security.ingest.hash.processor.key");
-        config.put("method", HashProcessor.Method.SHA1.toString());
-        ElasticsearchException e = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, "_tag", config));
-        assertThat(e.getMessage(), equalTo("[fields] a field-name entry is either empty or null"));
-    }
-
-    public void testProcessorMissingSalt() {
-        MockSecureSettings mockSecureSettings = new MockSecureSettings();
-        mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
-        Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
-        HashProcessor.Factory factory = new HashProcessor.Factory(settings);
-        Map<String, Object> config = new HashMap<>();
-        config.put("fields", Collections.singletonList("_field"));
-        config.put("target_field", "_target");
-        config.put("key_setting", "xpack.security.ingest.hash.processor.key");
-        ElasticsearchException e = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, "_tag", config));
-        assertThat(e.getMessage(), equalTo("[salt] required property is missing"));
-    }
-
-    public void testProcessorInvalidMethod() {
-        MockSecureSettings mockSecureSettings = new MockSecureSettings();
-        mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
-        Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
-        HashProcessor.Factory factory = new HashProcessor.Factory(settings);
-        Map<String, Object> config = new HashMap<>();
-        config.put("fields", Collections.singletonList("_field"));
-        config.put("salt", "_salt");
-        config.put("target_field", "_target");
-        config.put("key_setting", "xpack.security.ingest.hash.processor.key");
-        config.put("method", "invalid");
-        ElasticsearchException e = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, "_tag", config));
-        assertThat(e.getMessage(), equalTo("[method] type [invalid] not supported, cannot convert field. " +
-            "Valid hash methods: [sha1, sha256, sha384, sha512]"));
-    }
-
-    public void testProcessorInvalidOrMissingKeySetting() {
-        Settings settings = Settings.builder().setSecureSettings(new MockSecureSettings()).build();
-        HashProcessor.Factory factory = new HashProcessor.Factory(settings);
-        Map<String, Object> config = new HashMap<>();
-        config.put("fields", Collections.singletonList("_field"));
-        config.put("salt", "_salt");
-        config.put("target_field", "_target");
-        config.put("key_setting", "invalid");
-        config.put("method", HashProcessor.Method.SHA1.toString());
-        ElasticsearchException e = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, "_tag", new HashMap<>(config)));
-        assertThat(e.getMessage(),
-            equalTo("[key_setting] key [invalid] must match [xpack.security.ingest.hash.*.key]. It is not set"));
-        config.remove("key_setting");
-        ElasticsearchException ex = expectThrows(ElasticsearchException.class,
-            () -> factory.create(null, "_tag", config));
-        assertThat(ex.getMessage(), equalTo("[key_setting] required property is missing"));
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java
deleted file mode 100644
index b3890600592..00000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorTests.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.ingest;
-
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.security.ingest.HashProcessor.Method;
-
-import javax.crypto.Mac;
-import javax.crypto.SecretKeyFactory;
-import javax.crypto.spec.PBEKeySpec;
-import javax.crypto.spec.SecretKeySpec;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class HashProcessorTests extends ESTestCase {
-
-    @SuppressWarnings("unchecked")
-    public void testIgnoreMissing() throws Exception {
-        Method method = randomFrom(Method.values());
-        Mac mac = createMac(method);
-        Map<String, Object> fields = new HashMap<>();
-        fields.put("one", "foo");
-        HashProcessor processor = new HashProcessor("_tag", Arrays.asList("one", "two"),
-            "target", "_salt".getBytes(StandardCharsets.UTF_8), Method.SHA1, mac, true);
-        IngestDocument ingestDocument = new IngestDocument(fields, new HashMap<>());
-        processor.execute(ingestDocument);
-        Map<String, String> target = ingestDocument.getFieldValue("target", Map.class);
-        assertThat(target.size(), equalTo(1));
-        assertNotNull(target.get("one"));
-
-        HashProcessor failProcessor = new HashProcessor("_tag", Arrays.asList("one", "two"),
-            "target", "_salt".getBytes(StandardCharsets.UTF_8), Method.SHA1, mac, false);
-        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> failProcessor.execute(ingestDocument));
-        assertThat(exception.getMessage(), equalTo("field [two] not present as part of path [two]"));
-    }
-
-    public void testStaticKeyAndSalt() throws Exception {
-        byte[] salt = "_salt".getBytes(StandardCharsets.UTF_8);
-        SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
-        PBEKeySpec keySpec = new PBEKeySpec("hmackey".toCharArray(), salt, 5, 128);
-        byte[] pbkdf2 = secretKeyFactory.generateSecret(keySpec).getEncoded();
-        Mac mac = Mac.getInstance(Method.SHA1.getAlgorithm());
-        mac.init(new SecretKeySpec(pbkdf2, Method.SHA1.getAlgorithm()));
-        Map<String, Object> fields = new HashMap<>();
-        fields.put("field", "0123456789");
-        HashProcessor processor = new HashProcessor("_tag", Collections.singletonList("field"),
-            "target", salt, Method.SHA1, mac, false);
-        IngestDocument ingestDocument = new IngestDocument(fields, new HashMap<>());
-        processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue("target", String.class), equalTo("X3NhbHQMW0oHJGEEE9obGcGv5tGd7HFyDw=="));
-    }
-
-    public void testProcessorSingleField() throws Exception {
-        List<String> fields = Collections.singletonList(randomAlphaOfLength(6));
-        Map<String, Object> docFields = new HashMap<>();
-        for (String field : fields) {
-            docFields.put(field, randomAlphaOfLengthBetween(2, 10));
-        }
-
-        String targetField = randomAlphaOfLength(6);
-        Method method = randomFrom(Method.values());
-        Mac mac = createMac(method);
-        byte[] salt = randomByteArrayOfLength(5);
-        HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac, false);
-        IngestDocument ingestDocument = new IngestDocument(docFields, new HashMap<>());
-        processor.execute(ingestDocument);
-
-        String targetFieldValue = ingestDocument.getFieldValue(targetField, String.class);
-        Object expectedTargetFieldValue = method.hash(mac, salt, ingestDocument.getFieldValue(fields.get(0), String.class));
-        assertThat(targetFieldValue, equalTo(expectedTargetFieldValue));
-        byte[] bytes = Base64.getDecoder().decode(targetFieldValue);
-        byte[] actualSaltPrefix = new byte[salt.length];
-        System.arraycopy(bytes, 0, actualSaltPrefix, 0, salt.length);
-        assertArrayEquals(salt, actualSaltPrefix);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testProcessorMultipleFields() throws Exception {
-        List<String> fields = new ArrayList<>();
-        for (int i = 0; i < randomIntBetween(2, 10); i++) {
-            fields.add(randomAlphaOfLength(5 + i));
-        }
-        Map<String, Object> docFields = new HashMap<>();
-        for (String field : fields) {
-            docFields.put(field, randomAlphaOfLengthBetween(2, 10));
-        }
-
-        String targetField = randomAlphaOfLength(6);
-        Method method = randomFrom(Method.values());
-        Mac mac = createMac(method);
-        byte[] salt = randomByteArrayOfLength(5);
-        HashProcessor processor = new HashProcessor("_tag", fields, targetField, salt, method, mac, false);
-        IngestDocument ingestDocument = new IngestDocument(docFields, new HashMap<>());
-        processor.execute(ingestDocument);
-
-        Map<String, String> targetFieldMap = ingestDocument.getFieldValue(targetField, Map.class);
-        for (Map.Entry<String, String> entry : targetFieldMap.entrySet()) {
-            Object expectedTargetFieldValue = method.hash(mac, salt, ingestDocument.getFieldValue(entry.getKey(), String.class));
-            assertThat(entry.getValue(), equalTo(expectedTargetFieldValue));
-            byte[] bytes = Base64.getDecoder().decode(entry.getValue());
-            byte[] actualSaltPrefix = new byte[salt.length];
-            System.arraycopy(bytes, 0, actualSaltPrefix, 0, salt.length);
-            assertArrayEquals(salt, actualSaltPrefix);
-        }
-    }
-
-    private Mac createMac(Method method) throws Exception {
-        char[] password = randomAlphaOfLengthBetween(1, 10).toCharArray();
-        byte[] salt = randomAlphaOfLength(5).getBytes(StandardCharsets.UTF_8);
-        int iterations = randomIntBetween(1, 10);
-        SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2With" + method.getAlgorithm());
-        PBEKeySpec keySpec = new PBEKeySpec(password, salt, iterations, 128);
-        byte[] pbkdf2 = secretKeyFactory.generateSecret(keySpec).getEncoded();
-        Mac mac = Mac.getInstance(method.getAlgorithm());
-        mac.init(new SecretKeySpec(pbkdf2, method.getAlgorithm()));
-        return mac;
-    }
-}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml
deleted file mode 100644
index ee84e02d2f4..00000000000
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/hash_processor/10_basic.yml
+++ /dev/null
@@ -1,51 +0,0 @@
----
-teardown:
-  - do:
-      ingest.delete_pipeline:
-        id: "my_pipeline"
-        ignore: 404
-
----
-"Test Hash Processor":
-
-  - do:
-      cluster.health:
-        wait_for_status: yellow
-  - do:
-      ingest.put_pipeline:
-        id: "my_pipeline"
-        body:  >
-          {
-            "processors": [
-              {
-                "hash" : {
-                  "fields" : ["user_ssid"],
-                  "target_field" : "anonymized",
-                  "salt": "_salt",
-                  "iterations": 5,
-                  "method": "sha1",
-                  "key_setting": "xpack.security.ingest.hash.processor.key"
-                }
-              }
-            ]
-          }
-  - match: { acknowledged: true }
-
-  - do:
-      index:
-        index: test
-        type: test
-        id: 1
-        pipeline: "my_pipeline"
-        body:  >
-          {
-            "user_ssid": "0123456789"
-          }
-
-  - do:
-      get:
-        index: test
-        type: test
-        id: 1
-  - match: { _source.anonymized: "X3NhbHQMW0oHJGEEE9obGcGv5tGd7HFyDw==" }
-

From aae0133847254ce38dd2dc3ee01a10b486589243 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Wed, 18 Jul 2018 17:18:00 -0700
Subject: [PATCH 093/260] [test] port linux package packaging tests (#31943)

Add packaging tests for the linux package distributions to the java test
project and remove them from bats. Most of the tests that lived in
30_deb_package.bats and 40_rpm_package.bats are applicable to both
package types and are combined into a single type of test case. Others
are separated out into separate cases to make their intent more clear

For #26741
---
 qa/vagrant/build.gradle                       |  13 +-
 .../packaging/PackagingTests.java             |  22 +-
 .../test/DebPreservationTestCase.java         | 127 +++++++++
 .../packaging/test/DefaultDebBasicTests.java  |  31 +++
 .../test/DefaultDebPreservationTests.java     |  30 ++
 .../packaging/test/DefaultRpmBasicTests.java  |  30 ++
 .../test/DefaultRpmPreservationTests.java     |  30 ++
 .../packaging/test/OssDebBasicTests.java      |  30 ++
 .../test/OssDebPreservationTests.java         |  30 ++
 .../packaging/test/OssRpmBasicTests.java      |  30 ++
 .../test/OssRpmPreservationTests.java         |  30 ++
 .../test/PackageDependenciesTests.java        |  73 +++++
 .../packaging/test/PackageTestCase.java       | 168 ++++++++++++
 .../test/RpmPreservationTestCase.java         | 141 ++++++++++
 .../packaging/util/Archives.java              |  11 +-
 .../elasticsearch/packaging/util/Cleanup.java |  15 +-
 .../packaging/util/FileMatcher.java           |   1 +
 .../packaging/util/FileUtils.java             |  19 ++
 .../packaging/util/Installation.java          |  34 ++-
 .../packaging/util/Packages.java              | 259 ++++++++++++++++++
 .../packaging/util/Platforms.java             |  38 ++-
 .../packaging/util/ServerUtils.java           |   2 +-
 .../packaging/tests/30_deb_package.bats       | 233 ----------------
 .../packaging/tests/40_rpm_package.bats       | 220 ---------------
 24 files changed, 1114 insertions(+), 503 deletions(-)
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DebPreservationTestCase.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultDebBasicTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultDebPreservationTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultRpmBasicTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultRpmPreservationTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssDebBasicTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssDebPreservationTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssRpmBasicTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssRpmPreservationTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageDependenciesTests.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java
 create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java
 delete mode 100644 qa/vagrant/src/test/resources/packaging/tests/30_deb_package.bats
 delete mode 100644 qa/vagrant/src/test/resources/packaging/tests/40_rpm_package.bats

diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 37190632b44..4a0c9146962 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -28,7 +28,8 @@ plugins {
 
 dependencies {
   compile "junit:junit:${versions.junit}"
-  compile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+  compile "org.hamcrest:hamcrest-core:${versions.hamcrest}"
+  compile "org.hamcrest:hamcrest-library:${versions.hamcrest}"
   compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
 
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@@ -91,13 +92,5 @@ tasks.thirdPartyAudit.excludes = [
         'org.apache.log4j.Priority',
         // commons-logging provided dependencies
         'javax.servlet.ServletContextEvent',
-        'javax.servlet.ServletContextListener',
-        // from randomized testing
-        'org.apache.tools.ant.BuildException',
-        'org.apache.tools.ant.DirectoryScanner',
-        'org.apache.tools.ant.Task',
-        'org.apache.tools.ant.types.FileSet',
-        'org.easymock.EasyMock',
-        'org.easymock.IArgumentMatcher',
-        'org.jmock.core.Constraint'
+        'javax.servlet.ServletContextListener'
 ]
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java
index fa7f8e8ef78..57b647e1207 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/PackagingTests.java
@@ -19,10 +19,19 @@
 
 package org.elasticsearch.packaging;
 
+import org.elasticsearch.packaging.test.DefaultDebPreservationTests;
+import org.elasticsearch.packaging.test.DefaultDebBasicTests;
+import org.elasticsearch.packaging.test.DefaultRpmPreservationTests;
+import org.elasticsearch.packaging.test.DefaultRpmBasicTests;
+import org.elasticsearch.packaging.test.OssDebPreservationTests;
+import org.elasticsearch.packaging.test.OssDebBasicTests;
+import org.elasticsearch.packaging.test.OssRpmPreservationTests;
+import org.elasticsearch.packaging.test.OssRpmBasicTests;
 import org.elasticsearch.packaging.test.OssTarTests;
 import org.elasticsearch.packaging.test.OssZipTests;
 import org.elasticsearch.packaging.test.DefaultTarTests;
 import org.elasticsearch.packaging.test.DefaultZipTests;
+import org.elasticsearch.packaging.test.PackageDependenciesTests;
 
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
@@ -31,8 +40,17 @@ import org.junit.runners.Suite.SuiteClasses;
 @RunWith(Suite.class)
 @SuiteClasses({
     DefaultTarTests.class,
-    DefaultZipTests.class,
     OssTarTests.class,
-    OssZipTests.class
+    DefaultZipTests.class,
+    OssZipTests.class,
+    PackageDependenciesTests.class,
+    DefaultRpmBasicTests.class,
+    OssRpmBasicTests.class,
+    DefaultDebBasicTests.class,
+    OssDebBasicTests.class,
+    DefaultDebPreservationTests.class,
+    OssDebPreservationTests.class,
+    DefaultRpmPreservationTests.class,
+    OssRpmPreservationTests.class
 })
 public class PackagingTests {}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DebPreservationTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DebPreservationTestCase.java
new file mode 100644
index 00000000000..c584f5d2e44
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DebPreservationTestCase.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.Installation;
+import org.elasticsearch.packaging.util.Shell;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
+import static org.elasticsearch.packaging.util.FileUtils.assertPathsDontExist;
+import static org.elasticsearch.packaging.util.FileUtils.assertPathsExist;
+import static org.elasticsearch.packaging.util.Packages.SYSVINIT_SCRIPT;
+import static org.elasticsearch.packaging.util.Packages.assertInstalled;
+import static org.elasticsearch.packaging.util.Packages.assertRemoved;
+import static org.elasticsearch.packaging.util.Packages.install;
+import static org.elasticsearch.packaging.util.Packages.remove;
+import static org.elasticsearch.packaging.util.Packages.packageStatus;
+import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
+import static org.elasticsearch.packaging.util.Platforms.isDPKG;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeThat;
+import static org.junit.Assume.assumeTrue;
+
+@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
+public abstract class DebPreservationTestCase extends PackagingTestCase {
+
+    private static Installation installation;
+
+    protected abstract Distribution distribution();
+
+    @BeforeClass
+    public static void cleanup() {
+        installation = null;
+        cleanEverything();
+    }
+
+    @Before
+    public void onlyCompatibleDistributions() {
+        assumeTrue("only dpkg platforms", isDPKG());
+        assumeTrue("only compatible distributions", distribution().packaging.compatible);
+    }
+
+    public void test10Install() {
+        assertRemoved(distribution());
+        installation = install(distribution());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution());
+    }
+
+    public void test20Remove() {
+        assumeThat(installation, is(notNullValue()));
+
+        remove(distribution());
+
+        // some config files were not removed
+
+        assertPathsExist(
+            installation.config,
+            installation.config("elasticsearch.yml"),
+            installation.config("jvm.options"),
+            installation.config("log4j2.properties")
+        );
+
+        // keystore was removed
+
+        assertPathsDontExist(
+            installation.config("elasticsearch.keystore"),
+            installation.config(".elasticsearch.keystore.initial_md5sum")
+        );
+
+        // doc files were removed
+
+        assertPathsDontExist(
+            Paths.get("/usr/share/doc/" + distribution().flavor.name),
+            Paths.get("/usr/share/doc/" + distribution().flavor.name + "/copyright")
+        );
+
+        // sysvinit service file was not removed
+        assertTrue(Files.exists(SYSVINIT_SCRIPT));
+
+        // defaults file was not removed
+        assertTrue(Files.exists(installation.envFile));
+    }
+
+    public void test30Purge() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Shell sh = new Shell();
+        sh.run("dpkg --purge " + distribution().flavor.name);
+
+        assertRemoved(distribution());
+
+        assertPathsDontExist(
+            installation.config,
+            installation.envFile,
+            SYSVINIT_SCRIPT
+        );
+
+        assertThat(packageStatus(distribution()).exitCode, is(1));
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultDebBasicTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultDebBasicTests.java
new file mode 100644
index 00000000000..cd40c0e9e81
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultDebBasicTests.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Distribution;
+
+public class DefaultDebBasicTests extends PackageTestCase {
+
+    @Override
+    protected Distribution distribution() {
+        return Distribution.DEFAULT_DEB;
+    }
+
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultDebPreservationTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultDebPreservationTests.java
new file mode 100644
index 00000000000..d8b8c7f562b
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultDebPreservationTests.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Distribution;
+
+public class DefaultDebPreservationTests extends DebPreservationTestCase {
+
+    @Override
+    protected Distribution distribution() {
+        return Distribution.DEFAULT_DEB;
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultRpmBasicTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultRpmBasicTests.java
new file mode 100644
index 00000000000..a8ce7b48685
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultRpmBasicTests.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Distribution;
+
+public class DefaultRpmBasicTests extends PackageTestCase {
+
+    @Override
+    protected Distribution distribution() {
+        return Distribution.DEFAULT_RPM;
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultRpmPreservationTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultRpmPreservationTests.java
new file mode 100644
index 00000000000..633492cce6c
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultRpmPreservationTests.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Distribution;
+
+public class DefaultRpmPreservationTests extends RpmPreservationTestCase {
+
+    @Override
+    protected Distribution distribution() {
+        return Distribution.DEFAULT_RPM;
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssDebBasicTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssDebBasicTests.java
new file mode 100644
index 00000000000..5d779ac4653
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssDebBasicTests.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Distribution;
+
+public class OssDebBasicTests extends PackageTestCase {
+
+    @Override
+    protected Distribution distribution() {
+        return Distribution.OSS_DEB;
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssDebPreservationTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssDebPreservationTests.java
new file mode 100644
index 00000000000..cfce73bb160
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssDebPreservationTests.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Distribution;
+
+public class OssDebPreservationTests extends DebPreservationTestCase {
+
+    @Override
+    protected Distribution distribution() {
+        return Distribution.OSS_DEB;
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssRpmBasicTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssRpmBasicTests.java
new file mode 100644
index 00000000000..d3320c07955
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssRpmBasicTests.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Distribution;
+
+public class OssRpmBasicTests extends PackageTestCase {
+
+    @Override
+    protected Distribution distribution() {
+        return Distribution.OSS_RPM;
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssRpmPreservationTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssRpmPreservationTests.java
new file mode 100644
index 00000000000..87071d687d0
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssRpmPreservationTests.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Distribution;
+
+public class OssRpmPreservationTests extends RpmPreservationTestCase {
+
+    @Override
+    protected Distribution distribution() {
+        return Distribution.OSS_RPM;
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageDependenciesTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageDependenciesTests.java
new file mode 100644
index 00000000000..6861ef0a3ff
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageDependenciesTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Platforms;
+import org.elasticsearch.packaging.util.Shell;
+import org.elasticsearch.packaging.util.Shell.Result;
+
+import java.util.regex.Pattern;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.elasticsearch.packaging.util.Distribution.DEFAULT_DEB;
+import static org.elasticsearch.packaging.util.Distribution.DEFAULT_RPM;
+import static org.elasticsearch.packaging.util.Distribution.OSS_DEB;
+import static org.elasticsearch.packaging.util.Distribution.OSS_RPM;
+import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * Tests that linux packages correctly declare their dependencies and their conflicts
+ */
+public class PackageDependenciesTests extends PackagingTestCase {
+
+    public void testDebDependencies() {
+        assumeTrue(Platforms.isDPKG());
+
+        final Shell sh = new Shell();
+
+        final Result defaultResult = sh.run("dpkg -I " + getDistributionFile(DEFAULT_DEB));
+        final Result ossResult = sh.run("dpkg -I " + getDistributionFile(OSS_DEB));
+
+        assertTrue(Pattern.compile("(?m)^ Depends:.*bash.*").matcher(defaultResult.stdout).find());
+        assertTrue(Pattern.compile("(?m)^ Depends:.*bash.*").matcher(ossResult.stdout).find());
+
+        assertTrue(Pattern.compile("(?m)^ Conflicts: elasticsearch-oss$").matcher(defaultResult.stdout).find());
+        assertTrue(Pattern.compile("(?m)^ Conflicts: elasticsearch$").matcher(ossResult.stdout).find());
+    }
+
+    public void testRpmDependencies() {
+        assumeTrue(Platforms.isRPM());
+
+        final Shell sh = new Shell();
+
+        final Result defaultDeps = sh.run("rpm -qpR " + getDistributionFile(DEFAULT_RPM));
+        final Result ossDeps = sh.run("rpm -qpR " + getDistributionFile(OSS_RPM));
+
+        assertTrue(Pattern.compile("(?m)^/bin/bash\\s*$").matcher(defaultDeps.stdout).find());
+        assertTrue(Pattern.compile("(?m)^/bin/bash\\s*$").matcher(ossDeps.stdout).find());
+
+        final Result defaultConflicts = sh.run("rpm -qp --conflicts " + getDistributionFile(DEFAULT_RPM));
+        final Result ossConflicts = sh.run("rpm -qp --conflicts " + getDistributionFile(OSS_RPM));
+
+        assertTrue(Pattern.compile("(?m)^elasticsearch-oss\\s*$").matcher(defaultConflicts.stdout).find());
+        assertTrue(Pattern.compile("(?m)^elasticsearch\\s*$").matcher(ossConflicts.stdout).find());
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java
new file mode 100644
index 00000000000..28a767e95ae
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.Installation;
+import org.elasticsearch.packaging.util.Shell;
+
+import org.elasticsearch.packaging.util.Shell.Result;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
+import static org.elasticsearch.packaging.util.FileUtils.assertPathsDontExist;
+import static org.elasticsearch.packaging.util.Packages.SYSTEMD_SERVICE;
+import static org.elasticsearch.packaging.util.Packages.assertInstalled;
+import static org.elasticsearch.packaging.util.Packages.assertRemoved;
+import static org.elasticsearch.packaging.util.Packages.install;
+import static org.elasticsearch.packaging.util.Packages.remove;
+import static org.elasticsearch.packaging.util.Packages.startElasticsearch;
+import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
+import static org.elasticsearch.packaging.util.Platforms.getOsRelease;
+import static org.elasticsearch.packaging.util.Platforms.isSystemd;
+import static org.elasticsearch.packaging.util.ServerUtils.runElasticsearchTests;
+
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.isEmptyString;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assume.assumeThat;
+import static org.junit.Assume.assumeTrue;
+
+@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
+public abstract class PackageTestCase extends PackagingTestCase {
+
+    private static Installation installation;
+
+    protected abstract Distribution distribution();
+
+    @BeforeClass
+    public static void cleanup() {
+        installation = null;
+        cleanEverything();
+    }
+
+    @Before
+    public void onlyCompatibleDistributions() {
+        assumeTrue("only compatible distributions", distribution().packaging.compatible);
+    }
+
+    public void test10InstallPackage() {
+        assertRemoved(distribution());
+        installation = install(distribution());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution());
+    }
+
+    public void test20PluginsCommandWhenNoPlugins() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Shell sh = new Shell();
+        assertThat(sh.run(installation.bin("elasticsearch-plugin") + " list").stdout, isEmptyString());
+    }
+
+    public void test30InstallDoesNotStartServer() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Shell sh = new Shell();
+        assertThat(sh.run("ps aux").stdout, not(containsString("org.elasticsearch.bootstrap.Elasticsearch")));
+    }
+
+    public void test40StartServer() throws IOException {
+        assumeThat(installation, is(notNullValue()));
+
+        startElasticsearch();
+        runElasticsearchTests();
+        verifyPackageInstallation(installation, distribution()); // check startup script didn't change permissions
+    }
+
+    public void test50Remove() {
+        assumeThat(installation, is(notNullValue()));
+
+        remove(distribution());
+
+        // removing must stop the service
+        final Shell sh = new Shell();
+        assertThat(sh.run("ps aux").stdout, not(containsString("org.elasticsearch.bootstrap.Elasticsearch")));
+
+        if (isSystemd()) {
+
+            final int statusExitCode;
+
+            // Before version 231 systemctl returned exit code 3 for both services that were stopped, and nonexistent
+            // services [1]. In version 231 and later it returns exit code 4 for non-existent services.
+            //
+            // The exception is Centos 7 and oel 7 where it returns exit code 4 for non-existent services from a systemd reporting a version
+            // earlier than 231. Centos 6 does not have an /etc/os-release, but that's fine because it also doesn't use systemd.
+            //
+            // [1] https://github.com/systemd/systemd/pull/3385
+            if (getOsRelease().contains("ID=\"centos\"") || getOsRelease().contains("ID=\"ol\"")) {
+                statusExitCode = 4;
+            } else {
+
+                final Result versionResult = sh.run("systemctl --version");
+                final Matcher matcher = Pattern.compile("^systemd (\\d+)\n").matcher(versionResult.stdout);
+                matcher.find();
+                final int version = Integer.parseInt(matcher.group(1));
+
+                statusExitCode = version < 231
+                    ? 3
+                    : 4;
+            }
+
+            assertThat(sh.runIgnoreExitCode("systemctl status elasticsearch.service").exitCode, is(statusExitCode));
+            assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode, is(1));
+
+        }
+
+        assertPathsDontExist(
+            installation.bin,
+            installation.lib,
+            installation.modules,
+            installation.plugins,
+            installation.logs,
+            installation.pidDir
+        );
+
+        assertFalse(Files.exists(SYSTEMD_SERVICE));
+    }
+
+    public void test60Reinstall() {
+        assumeThat(installation, is(notNullValue()));
+
+        installation = install(distribution());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution());
+
+        remove(distribution());
+        assertRemoved(distribution());
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java
new file mode 100644
index 00000000000..527c1d2cc13
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.Installation;
+import org.elasticsearch.packaging.util.Shell;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
+import static org.elasticsearch.packaging.util.FileUtils.append;
+import static org.elasticsearch.packaging.util.FileUtils.assertPathsDontExist;
+import static org.elasticsearch.packaging.util.Packages.SYSTEMD_SERVICE;
+import static org.elasticsearch.packaging.util.Packages.SYSVINIT_SCRIPT;
+import static org.elasticsearch.packaging.util.Packages.assertInstalled;
+import static org.elasticsearch.packaging.util.Packages.assertRemoved;
+import static org.elasticsearch.packaging.util.Packages.install;
+import static org.elasticsearch.packaging.util.Packages.remove;
+import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
+import static org.elasticsearch.packaging.util.Platforms.isRPM;
+import static org.elasticsearch.packaging.util.Platforms.isSystemd;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeThat;
+import static org.junit.Assume.assumeTrue;
+
+@TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
+public abstract class RpmPreservationTestCase extends PackagingTestCase {
+
+    private static Installation installation;
+
+    protected abstract Distribution distribution();
+
+    @BeforeClass
+    public static void cleanup() {
+        installation = null;
+        cleanEverything();
+    }
+
+    @Before
+    public void onlyCompatibleDistributions() {
+        assumeTrue("only rpm platforms", isRPM());
+        assumeTrue("only compatible distributions", distribution().packaging.compatible);
+    }
+
+    public void test10Install() {
+        assertRemoved(distribution());
+        installation = install(distribution());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution());
+    }
+
+    public void test20Remove() {
+        assumeThat(installation, is(notNullValue()));
+
+        remove(distribution());
+
+        // config was removed
+        assertFalse(Files.exists(installation.config));
+
+        // sysvinit service file was removed
+        assertFalse(Files.exists(SYSVINIT_SCRIPT));
+
+        // defaults file was removed
+        assertFalse(Files.exists(installation.envFile));
+    }
+
+    public void test30PreserveConfig() {
+        final Shell sh = new Shell();
+
+        installation = install(distribution());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution());
+
+        sh.run("echo foobar | " + installation.executables().elasticsearchKeystore + " add --stdin foo.bar");
+        Stream.of(
+            installation.config("elasticsearch.yml"),
+            installation.config("jvm.options"),
+            installation.config("log4j2.properties")
+        ).forEach(path -> append(path, "# foo"));
+
+        remove(distribution());
+        assertRemoved(distribution());
+
+        if (isSystemd()) {
+            assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode, is(1));
+        }
+
+        assertPathsDontExist(
+            installation.bin,
+            installation.lib,
+            installation.modules,
+            installation.plugins,
+            installation.logs,
+            installation.pidDir,
+            installation.envFile,
+            SYSVINIT_SCRIPT,
+            SYSTEMD_SERVICE
+        );
+
+        assertTrue(Files.exists(installation.config));
+        assertTrue(Files.exists(installation.config("elasticsearch.keystore")));
+
+        Stream.of(
+            "elasticsearch.yml",
+            "jvm.options",
+            "log4j2.properties"
+        ).forEach(configFile -> {
+            final Path original = installation.config(configFile);
+            final Path saved = installation.config(configFile + ".rpmsave");
+            assertFalse(original + " should not exist", Files.exists(original));
+            assertTrue(saved + " should exist", Files.exists(saved));
+        });
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java
index 6ffec813eb0..9e9a453ca84 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java
@@ -35,7 +35,7 @@ import static org.elasticsearch.packaging.util.FileMatcher.p660;
 import static org.elasticsearch.packaging.util.FileMatcher.p755;
 import static org.elasticsearch.packaging.util.FileUtils.getCurrentVersion;
 import static org.elasticsearch.packaging.util.FileUtils.getDefaultArchiveInstallPath;
-import static org.elasticsearch.packaging.util.FileUtils.getPackagingArchivesDir;
+import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
 import static org.elasticsearch.packaging.util.FileUtils.lsGlob;
 
 import static org.elasticsearch.packaging.util.FileUtils.mv;
@@ -66,7 +66,7 @@ public class Archives {
     public static Installation installArchive(Distribution distribution, Path fullInstallPath, String version) {
         final Shell sh = new Shell();
 
-        final Path distributionFile = getPackagingArchivesDir().resolve(distribution.filename(version));
+        final Path distributionFile = getDistributionFile(distribution);
         final Path baseInstallPath = fullInstallPath.getParent();
         final Path extractedPath = baseInstallPath.resolve("elasticsearch-" + version);
 
@@ -106,7 +106,7 @@ public class Archives {
         Platforms.onLinux(() -> setupArchiveUsersLinux(fullInstallPath));
         Platforms.onWindows(() -> setupArchiveUsersWindows(fullInstallPath));
 
-        return new Installation(fullInstallPath);
+        return Installation.ofArchive(fullInstallPath);
     }
 
     private static void setupArchiveUsersLinux(Path installPath) {
@@ -176,7 +176,6 @@ public class Archives {
         ).forEach(dir -> assertThat(dir, file(Directory, owner, owner, p755)));
 
         assertThat(Files.exists(es.data), is(false));
-        assertThat(Files.exists(es.scripts), is(false));
 
         assertThat(es.bin, file(Directory, owner, owner, p755));
         assertThat(es.lib, file(Directory, owner, owner, p755));
@@ -209,7 +208,7 @@ public class Archives {
             "elasticsearch.yml",
             "jvm.options",
             "log4j2.properties"
-        ).forEach(config -> assertThat(es.config(config), file(File, owner, owner, p660)));
+        ).forEach(configFile -> assertThat(es.config(configFile), file(File, owner, owner, p660)));
 
         Stream.of(
             "NOTICE.txt",
@@ -252,7 +251,7 @@ public class Archives {
             "roles.yml",
             "role_mapping.yml",
             "log4j2.properties"
-        ).forEach(config -> assertThat(es.config(config), file(File, owner, owner, p660)));
+        ).forEach(configFile -> assertThat(es.config(configFile), file(File, owner, owner, p660)));
     }
 
     public static void runElasticsearch(Installation installation) throws IOException {
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java
index 4ff2998988c..fda61e9fb36 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java
@@ -27,11 +27,9 @@ import java.util.List;
 
 import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
 import static org.elasticsearch.packaging.util.FileUtils.lsGlob;
-import static org.elasticsearch.packaging.util.Platforms.isAptGet;
 import static org.elasticsearch.packaging.util.Platforms.isDPKG;
 import static org.elasticsearch.packaging.util.Platforms.isRPM;
 import static org.elasticsearch.packaging.util.Platforms.isSystemd;
-import static org.elasticsearch.packaging.util.Platforms.isYUM;
 
 public class Cleanup {
 
@@ -100,19 +98,14 @@ public class Cleanup {
         final Shell sh = new Shell();
 
         if (isRPM()) {
-            sh.runIgnoreExitCode("rpm --quiet -e elasticsearch elasticsearch-oss");
-        }
-
-        if (isYUM()) {
-            sh.runIgnoreExitCode("yum remove -y elasticsearch elasticsearch-oss");
+            // Doing rpm erase on both packages in one command will remove neither since both cannot be installed
+            // this may leave behind config files in /etc/elasticsearch, but a later step in this cleanup will get them
+            sh.runIgnoreExitCode("rpm --quiet -e elasticsearch");
+            sh.runIgnoreExitCode("rpm --quiet -e elasticsearch-oss");
         }
 
         if (isDPKG()) {
             sh.runIgnoreExitCode("dpkg --purge elasticsearch elasticsearch-oss");
         }
-
-        if (isAptGet()) {
-            sh.runIgnoreExitCode("apt-get --quiet --yes purge elasticsearch elasticsearch-oss");
-        }
     }
 }
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileMatcher.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileMatcher.java
index 9fdf6d60081..34bae68f97f 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileMatcher.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileMatcher.java
@@ -47,6 +47,7 @@ public class FileMatcher extends TypeSafeMatcher<Path> {
     public enum Fileness { File, Directory }
 
     public static final Set<PosixFilePermission> p755 = fromString("rwxr-xr-x");
+    public static final Set<PosixFilePermission> p750 = fromString("rwxr-x---");
     public static final Set<PosixFilePermission> p660 = fromString("rw-rw----");
     public static final Set<PosixFilePermission> p644 = fromString("rw-r--r--");
 
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java
index 315dc6ffee1..10d1b3ee6b6 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java
@@ -33,11 +33,14 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileOwnerAttributeView;
 import java.nio.file.attribute.PosixFileAttributes;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.IsNot.not;
 import static org.hamcrest.text.IsEmptyString.isEmptyOrNullString;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Wrappers and convenience methods for common filesystem operations
@@ -160,4 +163,20 @@ public class FileUtils {
         assertThat(fromEnv, not(isEmptyOrNullString()));
         return Paths.get(fromEnv);
     }
+
+    public static Path getDistributionFile(Distribution distribution) {
+        return getDistributionFile(distribution, getCurrentVersion());
+    }
+
+    public static Path getDistributionFile(Distribution distribution, String version) {
+        return getPackagingArchivesDir().resolve(distribution.filename(version));
+    }
+
+    public static void assertPathsExist(Path... paths) {
+        Arrays.stream(paths).forEach(path -> assertTrue(path + " should exist", Files.exists(path)));
+    }
+
+    public static void assertPathsDontExist(Path... paths) {
+        Arrays.stream(paths).forEach(path -> assertFalse(path + " should not exist", Files.exists(path)));
+    }
 }
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java
index 68da440400a..40dc546f230 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.packaging.util;
 
 import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
  * Represents an installation of Elasticsearch
@@ -34,9 +35,10 @@ public class Installation {
     public final Path logs;
     public final Path plugins;
     public final Path modules;
-    public final Path scripts;
+    public final Path pidDir;
+    public final Path envFile;
 
-    public Installation(Path home, Path config, Path data, Path logs, Path plugins, Path modules, Path scripts) {
+    public Installation(Path home, Path config, Path data, Path logs, Path plugins, Path modules, Path pidDir, Path envFile) {
         this.home = home;
         this.bin = home.resolve("bin");
         this.lib = home.resolve("lib");
@@ -46,18 +48,38 @@ public class Installation {
         this.logs = logs;
         this.plugins = plugins;
         this.modules = modules;
-        this.scripts = scripts;
+        this.pidDir = pidDir;
+        this.envFile = envFile;
     }
 
-    public Installation(Path home) {
-        this(
+    public static Installation ofArchive(Path home) {
+        return new Installation(
             home,
             home.resolve("config"),
             home.resolve("data"),
             home.resolve("logs"),
             home.resolve("plugins"),
             home.resolve("modules"),
-            home.resolve("scripts")
+            null,
+            null
+        );
+    }
+
+    public static Installation ofPackage(Distribution.Packaging packaging) {
+
+        final Path envFile = (packaging == Distribution.Packaging.RPM)
+            ? Paths.get("/etc/sysconfig/elasticsearch")
+            : Paths.get("/etc/default/elasticsearch");
+
+        return new Installation(
+            Paths.get("/usr/share/elasticsearch"),
+            Paths.get("/etc/elasticsearch"),
+            Paths.get("/var/lib/elasticsearch"),
+            Paths.get("/var/log/elasticsearch"),
+            Paths.get("/usr/share/elasticsearch/plugins"),
+            Paths.get("/usr/share/elasticsearch/modules"),
+            Paths.get("/var/run/elasticsearch"),
+            envFile
         );
     }
 
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java
new file mode 100644
index 00000000000..6e80d9e027d
--- /dev/null
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.packaging.util;
+
+import org.elasticsearch.packaging.util.Shell.Result;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.packaging.util.FileMatcher.Fileness.Directory;
+import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
+import static org.elasticsearch.packaging.util.FileMatcher.file;
+import static org.elasticsearch.packaging.util.FileMatcher.p644;
+import static org.elasticsearch.packaging.util.FileMatcher.p660;
+import static org.elasticsearch.packaging.util.FileMatcher.p750;
+import static org.elasticsearch.packaging.util.FileMatcher.p755;
+import static org.elasticsearch.packaging.util.FileUtils.getCurrentVersion;
+import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
+import static org.elasticsearch.packaging.util.Platforms.isSysVInit;
+import static org.elasticsearch.packaging.util.Platforms.isSystemd;
+import static org.elasticsearch.packaging.util.ServerUtils.waitForElasticsearch;
+import static org.hamcrest.CoreMatchers.anyOf;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class Packages {
+
+    public static final Path SYSVINIT_SCRIPT = Paths.get("/etc/init.d/elasticsearch");
+    public static final Path SYSTEMD_SERVICE = Paths.get("/usr/lib/systemd/system/elasticsearch.service");
+
+    public static void assertInstalled(Distribution distribution) {
+        final Result status = packageStatus(distribution);
+        assertThat(status.exitCode, is(0));
+
+        Platforms.onDPKG(() -> assertFalse(Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find()));
+    }
+
+    public static void assertRemoved(Distribution distribution) {
+        final Result status = packageStatus(distribution);
+
+        Platforms.onRPM(() -> assertThat(status.exitCode, is(1)));
+
+        Platforms.onDPKG(() -> {
+            assertThat(status.exitCode, anyOf(is(0), is(1)));
+            if (status.exitCode == 0) {
+                assertTrue(Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find());
+            }
+        });
+    }
+
+    public static Result packageStatus(Distribution distribution) {
+        final Shell sh = new Shell();
+        final Result result;
+
+        if (distribution.packaging == Distribution.Packaging.RPM) {
+            result = sh.runIgnoreExitCode("rpm -qe " + distribution.flavor.name);
+        } else {
+            result = sh.runIgnoreExitCode("dpkg -s " + distribution.flavor.name);
+        }
+
+        return result;
+    }
+
+    public static Installation install(Distribution distribution) {
+        return install(distribution, getCurrentVersion());
+    }
+
+    public static Installation install(Distribution distribution, String version) {
+        final Shell sh = new Shell();
+        final Path distributionFile = getDistributionFile(distribution, version);
+
+        Platforms.onRPM(() -> sh.run("rpm -i " + distributionFile));
+        Platforms.onDPKG(() -> sh.run("dpkg -i " + distributionFile));
+
+        return Installation.ofPackage(distribution.packaging);
+    }
+
+    public static void remove(Distribution distribution) {
+        final Shell sh = new Shell();
+
+        Platforms.onRPM(() -> {
+            sh.run("rpm -e " + distribution.flavor.name);
+            final Result status = packageStatus(distribution);
+            assertThat(status.exitCode, is(1));
+        });
+
+        Platforms.onDPKG(() -> {
+            sh.run("dpkg -r " + distribution.flavor.name);
+            final Result status = packageStatus(distribution);
+            assertThat(status.exitCode, is(0));
+            assertTrue(Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find());
+        });
+    }
+
+    public static void verifyPackageInstallation(Installation installation, Distribution distribution) {
+        verifyOssInstallation(installation, distribution);
+        if (distribution.flavor == Distribution.Flavor.DEFAULT) {
+            verifyDefaultInstallation(installation);
+        }
+    }
+
+
+    private static void verifyOssInstallation(Installation es, Distribution distribution) {
+        final Shell sh = new Shell();
+
+        sh.run("id elasticsearch");
+        sh.run("getent group elasticsearch");
+
+        final Result passwdResult = sh.run("getent passwd elasticsearch");
+        final Path homeDir = Paths.get(passwdResult.stdout.trim().split(":")[5]);
+        assertFalse("elasticsearch user home directory must not exist", Files.exists(homeDir));
+
+        Stream.of(
+            es.home,
+            es.plugins,
+            es.modules
+        ).forEach(dir -> assertThat(dir, file(Directory, "root", "root", p755)));
+
+        assertThat(es.pidDir, file(Directory, "elasticsearch", "elasticsearch", p755));
+
+        Stream.of(
+            es.data,
+            es.logs
+        ).forEach(dir -> assertThat(dir, file(Directory, "elasticsearch", "elasticsearch", p750)));
+
+        // we shell out here because java's posix file permission view doesn't support special modes
+        assertThat(es.config, file(Directory, "root", "elasticsearch", p750));
+        assertThat(sh.run("find \"" + es.config + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750"));
+
+        Stream.of(
+            "elasticsearch.keystore",
+            "elasticsearch.yml",
+            "jvm.options",
+            "log4j2.properties"
+        ).forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660)));
+        assertThat(es.config(".elasticsearch.keystore.initial_md5sum"), file(File, "root", "elasticsearch", p644));
+
+        assertThat(sh.run("sudo -u elasticsearch " + es.bin("elasticsearch-keystore") + " list").stdout, containsString("keystore.seed"));
+
+        Stream.of(
+            es.bin,
+            es.lib
+        ).forEach(dir -> assertThat(dir, file(Directory, "root", "root", p755)));
+
+        Stream.of(
+            "elasticsearch",
+            "elasticsearch-plugin",
+            "elasticsearch-keystore",
+            "elasticsearch-translog"
+        ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
+
+        Stream.of(
+            "NOTICE.txt",
+            "README.textile"
+        ).forEach(doc -> assertThat(es.home.resolve(doc), file(File, "root", "root", p644)));
+
+        assertThat(es.envFile, file(File, "root", "elasticsearch", p660));
+
+        if (distribution.packaging == Distribution.Packaging.RPM) {
+            assertThat(es.home.resolve("LICENSE.txt"), file(File, "root", "root", p644));
+        } else {
+            Path copyrightDir = Paths.get(sh.run("readlink -f /usr/share/doc/" + distribution.flavor.name).stdout.trim());
+            assertThat(copyrightDir, file(Directory, "root", "root", p755));
+            assertThat(copyrightDir.resolve("copyright"), file(File, "root", "root", p644));
+        }
+
+        if (isSystemd()) {
+            Stream.of(
+                SYSTEMD_SERVICE,
+                Paths.get("/usr/lib/tmpfiles.d/elasticsearch.conf"),
+                Paths.get("/usr/lib/sysctl.d/elasticsearch.conf")
+            ).forEach(confFile -> assertThat(confFile, file(File, "root", "root", p644)));
+
+            final String sysctlExecutable = (distribution.packaging == Distribution.Packaging.RPM)
+                ? "/usr/sbin/sysctl"
+                : "/sbin/sysctl";
+            assertThat(sh.run(sysctlExecutable + " vm.max_map_count").stdout, containsString("vm.max_map_count = 262144"));
+        }
+
+        if (isSysVInit()) {
+            assertThat(SYSVINIT_SCRIPT, file(File, "root", "root", p750));
+        }
+    }
+
+    private static void verifyDefaultInstallation(Installation es) {
+
+        Stream.of(
+            "elasticsearch-certgen",
+            "elasticsearch-certutil",
+            "elasticsearch-croneval",
+            "elasticsearch-migrate",
+            "elasticsearch-saml-metadata",
+            "elasticsearch-setup-passwords",
+            "elasticsearch-sql-cli",
+            "elasticsearch-syskeygen",
+            "elasticsearch-users",
+            "x-pack-env",
+            "x-pack-security-env",
+            "x-pack-watcher-env"
+        ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
+
+        // at this time we only install the current version of archive distributions, but if that changes we'll need to pass
+        // the version through here
+        assertThat(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, "root", "root", p755));
+
+        Stream.of(
+            "users",
+            "users_roles",
+            "roles.yml",
+            "role_mapping.yml",
+            "log4j2.properties"
+        ).forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660)));
+    }
+
+    public static void startElasticsearch() throws IOException {
+        final Shell sh = new Shell();
+        if (isSystemd()) {
+            sh.run("systemctl daemon-reload");
+            sh.run("systemctl enable elasticsearch.service");
+            sh.run("systemctl is-enabled elasticsearch.service");
+            sh.run("systemctl start elasticsearch.service");
+        } else {
+            sh.run("service elasticsearch start");
+        }
+
+        waitForElasticsearch();
+
+        if (isSystemd()) {
+            sh.run("systemctl is-active elasticsearch.service");
+            sh.run("systemctl status elasticsearch.service");
+        } else {
+            sh.run("service elasticsearch status");
+        }
+    }
+}
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java
index 5ffbc318200..c7ca1284ca6 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java
@@ -19,11 +19,23 @@
 
 package org.elasticsearch.packaging.util;
 
+import java.nio.file.Paths;
+
+import static org.elasticsearch.packaging.util.FileUtils.slurp;
+
 public class Platforms {
     public static final String OS_NAME = System.getProperty("os.name");
     public static final boolean LINUX = OS_NAME.startsWith("Linux");
     public static final boolean WINDOWS = OS_NAME.startsWith("Windows");
 
+    public static String getOsRelease() {
+        if (LINUX) {
+            return slurp(Paths.get("/etc/os-release"));
+        } else {
+            throw new RuntimeException("os-release is only supported on linux");
+        }
+    }
+
     public static boolean isDPKG() {
         if (WINDOWS) {
             return false;
@@ -31,13 +43,6 @@ public class Platforms {
         return new Shell().runIgnoreExitCode("which dpkg").isSuccess();
     }
 
-    public static boolean isAptGet() {
-        if (WINDOWS) {
-            return false;
-        }
-        return new Shell().runIgnoreExitCode("which apt-get").isSuccess();
-    }
-
     public static boolean isRPM() {
         if (WINDOWS) {
             return false;
@@ -45,13 +50,6 @@ public class Platforms {
         return new Shell().runIgnoreExitCode("which rpm").isSuccess();
     }
 
-    public static boolean isYUM() {
-        if (WINDOWS) {
-            return false;
-        }
-        return new Shell().runIgnoreExitCode("which yum").isSuccess();
-    }
-
     public static boolean isSystemd() {
         if (WINDOWS) {
             return false;
@@ -78,6 +76,18 @@ public class Platforms {
         }
     }
 
+    public static void onRPM(PlatformAction action) {
+        if (isRPM()) {
+            action.run();
+        }
+    }
+
+    public static void onDPKG(PlatformAction action) {
+        if (isDPKG()) {
+            action.run();
+        }
+    }
+
     /**
      * Essentially a Runnable, but we make the distinction so it's more clear that these are synchronous
      */
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java
index ff006a34e68..6331b4bf46e 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java
@@ -72,7 +72,7 @@ public class ServerUtils {
 
             } catch (HttpHostConnectException e) {
                 // we want to retry if the connection is refused
-                LOG.info("Got connection refused when waiting for cluster health", e);
+                LOG.debug("Got connection refused when waiting for cluster health", e);
             }
 
             timeElapsed = System.currentTimeMillis() - startTime;
diff --git a/qa/vagrant/src/test/resources/packaging/tests/30_deb_package.bats b/qa/vagrant/src/test/resources/packaging/tests/30_deb_package.bats
deleted file mode 100644
index 749c72c8b31..00000000000
--- a/qa/vagrant/src/test/resources/packaging/tests/30_deb_package.bats
+++ /dev/null
@@ -1,233 +0,0 @@
-#!/usr/bin/env bats
-
-# This file is used to test the installation and removal
-# of a Debian package.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It should only be executed
-# in a throw-away VM like those made by the Vagrantfile at
-# the root of the Elasticsearch source code. This should
-# cause the script to fail if it is executed any other way:
-[ -f /etc/is_vagrant_vm ] || {
-  >&2 echo "must be run on a vagrant VM"
-  exit 1
-}
-
-# The test case can be executed with the Bash Automated
-# Testing System tool available at https://github.com/sstephenson/bats
-# Thanks to Sam Stephenson!
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Load test utilities
-load $BATS_UTILS/utils.bash
-load $BATS_UTILS/packages.bash
-load $BATS_UTILS/plugins.bash
-
-# Cleans everything for the 1st execution
-setup() {
-    skip_not_dpkg
-    export_elasticsearch_paths
-}
-
-@test "[DEB] package depends on bash" {
-    dpkg -I elasticsearch-oss-$(cat version).deb | grep "Depends:.*bash.*"
-}
-
-@test "[DEB] package conflicts" {
-    dpkg -I elasticsearch-oss-$(cat version).deb | grep "^ Conflicts: elasticsearch$"
-    dpkg -I elasticsearch-$(cat version).deb | grep "^ Conflicts: elasticsearch-oss$"
-}
-
-##################################
-# Install DEB package
-##################################
-@test "[DEB] dpkg command is available" {
-    clean_before_test
-    dpkg --version
-}
-
-@test "[DEB] package is available" {
-    count=$(ls elasticsearch-oss-$(cat version).deb | wc -l)
-    [ "$count" -eq 1 ]
-}
-
-@test "[DEB] package is not installed" {
-    run dpkg -s 'elasticsearch-oss'
-    [ "$status" -eq 1 ]
-}
-
-@test "[DEB] temporarily remove java and ensure the install fails" {
-    move_java
-    run dpkg -i elasticsearch-oss-$(cat version).deb
-    output=$status
-    unmove_java
-    [ "$output" -eq 1 ]
-}
-
-@test "[DEB] install package" {
-    dpkg -i elasticsearch-oss-$(cat version).deb
-}
-
-@test "[DEB] package is installed" {
-    dpkg -s 'elasticsearch-oss'
-}
-
-@test "[DEB] verify package installation" {
-    verify_package_installation
-}
-
-@test "[DEB] verify elasticsearch-plugin list runs without any plugins installed" {
-    local plugins_list=`$ESHOME/bin/elasticsearch-plugin list`
-    [[ -z $plugins_list ]]
-}
-
-@test "[DEB] elasticsearch isn't started by package install" {
-    # Wait a second to give Elasticsearch a change to start if it is going to.
-    # This isn't perfect by any means but its something.
-    sleep 1
-    ! ps aux | grep elasticsearch | grep java
-    # You might be tempted to use jps instead of the above but that'd have to
-    # look like:
-    # ! sudo -u elasticsearch jps | grep -i elasticsearch
-    # which isn't really easier to read than the above.
-}
-
-@test "[DEB] test elasticsearch" {
-    start_elasticsearch_service
-    run_elasticsearch_tests
-}
-
-@test "[DEB] verify package installation after start" {
-    # Checks that the startup scripts didn't change the permissions
-    verify_package_installation
-}
-
-##################################
-# Uninstall DEB package
-##################################
-@test "[DEB] remove package" {
-    dpkg -r 'elasticsearch-oss'
-}
-
-@test "[DEB] package has been removed" {
-    run dpkg -s 'elasticsearch-oss'
-    [ "$status" -eq 0 ]
-    echo "$output" | grep -i "status" | grep -i "deinstall ok"
-}
-
-@test "[DEB] verify package removal" {
-    # The removal must stop the service
-    count=$(ps | grep Elasticsearch | wc -l)
-    [ "$count" -eq 0 ]
-
-    # The removal must disable the service
-    # see prerm file
-    if is_systemd; then
-        missing_exit_code=4
-        if [ $(systemctl --version | head -1 | awk '{print $2}') -lt 231 ]; then
-          # systemd before version 231 used exit code 3 when the service did not exist
-          missing_exit_code=3
-        fi
-        run systemctl status elasticsearch.service
-        [ "$status" -eq $missing_exit_code ]
-
-        run systemctl is-enabled elasticsearch.service
-        [ "$status" -eq 1 ]
-    fi
-
-    # Those directories are deleted when removing the package
-    # see postrm file
-    assert_file_not_exist "/var/log/elasticsearch"
-    assert_file_not_exist "/usr/share/elasticsearch/plugins"
-    assert_file_not_exist "/usr/share/elasticsearch/modules"
-    assert_file_not_exist "/var/run/elasticsearch"
-
-    # Those directories are removed by the package manager
-    assert_file_not_exist "/usr/share/elasticsearch/bin"
-    assert_file_not_exist "/usr/share/elasticsearch/lib"
-    assert_file_not_exist "/usr/share/elasticsearch/modules"
-    assert_file_not_exist "/usr/share/elasticsearch/modules/lang-painless"
-
-    # The configuration files are still here
-    assert_file_exist "/etc/elasticsearch"
-    # TODO: use ucf to handle these better for Debian-based systems
-    assert_file_not_exist "/etc/elasticsearch/elasticsearch.keystore"
-    assert_file_not_exist "/etc/elasticsearch/.elasticsearch.keystore.initial_md5sum"
-    assert_file_exist "/etc/elasticsearch/elasticsearch.yml"
-    assert_file_exist "/etc/elasticsearch/jvm.options"
-    assert_file_exist "/etc/elasticsearch/log4j2.properties"
-
-    # The env file is still here
-    assert_file_exist "/etc/default/elasticsearch"
-
-    # The service files are still here
-    assert_file_exist "/etc/init.d/elasticsearch"
-}
-
-@test "[DEB] purge package" {
-    # User installed scripts aren't removed so we'll just get them ourselves
-    rm -rf $ESSCRIPTS
-    dpkg --purge 'elasticsearch-oss'
-}
-
-@test "[DEB] verify package purge" {
-    # all remaining files are deleted by the purge
-    assert_file_not_exist "/etc/elasticsearch"
-    assert_file_not_exist "/etc/elasticsearch/elasticsearch.keystore"
-    assert_file_not_exist "/etc/elasticsearch/.elasticsearch.keystore.initial_md5sum"
-    assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
-    assert_file_not_exist "/etc/elasticsearch/jvm.options"
-    assert_file_not_exist "/etc/elasticsearch/log4j2.properties"
-
-    assert_file_not_exist "/etc/default/elasticsearch"
-
-    assert_file_not_exist "/etc/init.d/elasticsearch"
-    assert_file_not_exist "/usr/lib/systemd/system/elasticsearch.service"
-
-    assert_file_not_exist "/usr/share/elasticsearch"
-
-    assert_file_not_exist "/usr/share/doc/elasticsearch-oss"
-    assert_file_not_exist "/usr/share/doc/elasticsearch-oss/copyright"
-}
-
-@test "[DEB] package has been completly removed" {
-    run dpkg -s 'elasticsearch-oss'
-    [ "$status" -eq 1 ]
-}
-
-@test "[DEB] reinstall package" {
-    dpkg -i elasticsearch-oss-$(cat version).deb
-}
-
-@test "[DEB] package is installed by reinstall" {
-    dpkg -s 'elasticsearch-oss'
-}
-
-@test "[DEB] verify package reinstallation" {
-    verify_package_installation
-}
-
-@test "[DEB] repurge package" {
-    dpkg --purge 'elasticsearch-oss'
-}
-
-@test "[DEB] package has been completly removed again" {
-    run dpkg -s 'elasticsearch-oss'
-    [ "$status" -eq 1 ]
-}
diff --git a/qa/vagrant/src/test/resources/packaging/tests/40_rpm_package.bats b/qa/vagrant/src/test/resources/packaging/tests/40_rpm_package.bats
deleted file mode 100644
index cb12d4b50e0..00000000000
--- a/qa/vagrant/src/test/resources/packaging/tests/40_rpm_package.bats
+++ /dev/null
@@ -1,220 +0,0 @@
-#!/usr/bin/env bats
-
-# This file is used to test the installation of a RPM package.
-
-# WARNING: This testing file must be executed as root and can
-# dramatically change your system. It should only be executed
-# in a throw-away VM like those made by the Vagrantfile at
-# the root of the Elasticsearch source code. This should
-# cause the script to fail if it is executed any other way:
-[ -f /etc/is_vagrant_vm ] || {
-  >&2 echo "must be run on a vagrant VM"
-  exit 1
-}
-
-# The test case can be executed with the Bash Automated
-# Testing System tool available at https://github.com/sstephenson/bats
-# Thanks to Sam Stephenson!
-
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Load test utilities
-load $BATS_UTILS/utils.bash
-load $BATS_UTILS/packages.bash
-load $BATS_UTILS/plugins.bash
-
-# Cleans everything for the 1st execution
-setup() {
-    skip_not_rpm
-    export_elasticsearch_paths
-}
-
-@test "[RPM] package depends on bash" {
-    rpm -qpR elasticsearch-oss-$(cat version).rpm | grep '/bin/bash'
-}
-
-@test "[RPM] package conflicts" {
-    rpm -qp --conflicts elasticsearch-oss-$(cat version).rpm | grep "^elasticsearch\s*$"
-    rpm -qp --conflicts elasticsearch-$(cat version).rpm | grep "^elasticsearch-oss\s*$"
-}
-
-##################################
-# Install RPM package
-##################################
-@test "[RPM] rpm command is available" {
-    clean_before_test
-    rpm --version
-}
-
-@test "[RPM] package is available" {
-    count=$(ls elasticsearch-oss-$(cat version).rpm | wc -l)
-    [ "$count" -eq 1 ]
-}
-
-@test "[RPM] package is not installed" {
-    run rpm -qe 'elasticsearch-oss'
-    [ "$status" -eq 1 ]
-}
-
-@test "[RPM] temporarily remove java and ensure the install fails" {
-    move_java
-    run rpm -i elasticsearch-oss-$(cat version).rpm
-    output=$status
-    unmove_java
-    [ "$output" -eq 1 ]
-}
-
-@test "[RPM] install package" {
-    rpm -i elasticsearch-oss-$(cat version).rpm
-}
-
-@test "[RPM] package is installed" {
-    rpm -qe 'elasticsearch-oss'
-}
-
-@test "[RPM] verify package installation" {
-    verify_package_installation
-}
-
-@test "[RPM] verify elasticsearch-plugin list runs without any plugins installed" {
-    local plugins_list=`$ESHOME/bin/elasticsearch-plugin list`
-    [[ -z $plugins_list ]]
-}
-
-@test "[RPM] elasticsearch isn't started by package install" {
-    # Wait a second to give Elasticsearch a change to start if it is going to.
-    # This isn't perfect by any means but its something.
-    sleep 1
-    ! ps aux | grep elasticsearch | grep java
-}
-
-@test "[RPM] test elasticsearch" {
-    start_elasticsearch_service
-    run_elasticsearch_tests
-}
-
-@test "[RPM] verify package installation after start" {
-    # Checks that the startup scripts didn't change the permissions
-    verify_package_installation
-}
-
-@test "[RPM] remove package" {
-    # User installed scripts aren't removed so we'll just get them ourselves
-    rm -rf $ESSCRIPTS
-    rpm -e 'elasticsearch-oss'
-}
-
-@test "[RPM] package has been removed" {
-    run rpm -qe 'elasticsearch-oss'
-    [ "$status" -eq 1 ]
-}
-
-@test "[RPM] verify package removal" {
-    # The removal must stop the service
-    count=$(ps | grep Elasticsearch | wc -l)
-    [ "$count" -eq 0 ]
-
-    # The removal must disable the service
-    # see prerm file
-    if is_systemd; then
-        run systemctl is-enabled elasticsearch.service
-        [ "$status" -eq 1 ]
-    fi
-
-    # Those directories are deleted when removing the package
-    # see postrm file
-    assert_file_not_exist "/var/log/elasticsearch"
-    assert_file_not_exist "/usr/share/elasticsearch/plugins"
-    assert_file_not_exist "/var/run/elasticsearch"
-
-    # Those directories are removed by the package manager
-    assert_file_not_exist "/usr/share/elasticsearch/bin"
-    assert_file_not_exist "/usr/share/elasticsearch/lib"
-    assert_file_not_exist "/usr/share/elasticsearch/modules"
-
-    assert_file_not_exist "/etc/elasticsearch"
-
-    assert_file_not_exist "/etc/init.d/elasticsearch"
-    assert_file_not_exist "/usr/lib/systemd/system/elasticsearch.service"
-
-    assert_file_not_exist "/etc/sysconfig/elasticsearch"
-}
-
-@test "[RPM] reinstall package" {
-    rpm -i elasticsearch-oss-$(cat version).rpm
-}
-
-@test "[RPM] package is installed by reinstall" {
-    rpm -qe 'elasticsearch-oss'
-}
-
-@test "[RPM] verify package reinstallation" {
-    verify_package_installation
-}
-
-@test "[RPM] reremove package" {
-    echo foobar | "$ESHOME/bin/elasticsearch-keystore" add --stdin foo.bar
-    echo "# ping" >> "/etc/elasticsearch/elasticsearch.yml"
-    echo "# ping" >> "/etc/elasticsearch/jvm.options"
-    echo "# ping" >> "/etc/elasticsearch/log4j2.properties"
-    rpm -e 'elasticsearch-oss'
-}
-
-@test "[RPM] verify preservation" {
-    # The removal must disable the service
-    # see prerm file
-    if is_systemd; then
-        run systemctl is-enabled elasticsearch.service
-        [ "$status" -eq 1 ]
-    fi
-
-    # Those directories are deleted when removing the package
-    # see postrm file
-    assert_file_not_exist "/var/log/elasticsearch"
-    assert_file_not_exist "/usr/share/elasticsearch/plugins"
-    assert_file_not_exist "/usr/share/elasticsearch/modules"
-    assert_file_not_exist "/var/run/elasticsearch"
-
-    assert_file_not_exist "/usr/share/elasticsearch/bin"
-    assert_file_not_exist "/usr/share/elasticsearch/lib"
-    assert_file_not_exist "/usr/share/elasticsearch/modules"
-    assert_file_not_exist "/usr/share/elasticsearch/modules/lang-painless"
-
-    assert_file_exist "/etc/elasticsearch/elasticsearch.keystore"
-    assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
-    assert_file_exist "/etc/elasticsearch/elasticsearch.yml.rpmsave"
-    assert_file_not_exist "/etc/elasticsearch/jvm.options"
-    assert_file_exist "/etc/elasticsearch/jvm.options.rpmsave"
-    assert_file_not_exist "/etc/elasticsearch/log4j2.properties"
-    assert_file_exist "/etc/elasticsearch/log4j2.properties.rpmsave"
-
-    assert_file_not_exist "/etc/init.d/elasticsearch"
-    assert_file_not_exist "/usr/lib/systemd/system/elasticsearch.service"
-
-    assert_file_not_exist "/etc/sysconfig/elasticsearch"
-}
-
-@test "[RPM] finalize package removal" {
-    # cleanup
-    rm -rf /etc/elasticsearch
-}
-
-@test "[RPM] package has been removed again" {
-    run rpm -qe 'elasticsearch-oss'
-    [ "$status" -eq 1 ]
-}

From 38e2e1d553280de0f49ed9bb5a2c7df4187d4ed5 Mon Sep 17 00:00:00 2001
From: Alpar Torok <torokalpar@gmail.com>
Date: Thu, 19 Jul 2018 06:46:58 +0000
Subject: [PATCH 094/260] Detect and prevent configuration that triggers a
 Gradle bug (#31912)

* Detect and prevent configuration that triggers a Gradle bug

As we found in #31862, this can lead to a lot of wasted time as it's not
immediatly obvius what's going on.
Givent how many projects we have it's getting increasingly easier to run
into gradle/gradle#847.
---
 build.gradle                                    | 17 ++++++++++++++++-
 client/test/build.gradle                        |  2 ++
 libs/cli/build.gradle                           | 10 ----------
 plugins/discovery-ec2/qa/build.gradle           |  1 +
 plugins/repository-azure/qa/build.gradle        |  1 +
 plugins/repository-gcs/qa/build.gradle          |  1 +
 settings.gradle                                 |  2 ++
 .../with-system-key/build.gradle                |  1 +
 .../without-system-key/build.gradle             |  1 +
 x-pack/qa/sql/security/build.gradle             |  2 ++
 10 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/build.gradle b/build.gradle
index 90a9d88ac8b..66f34d8f445 100644
--- a/build.gradle
+++ b/build.gradle
@@ -222,7 +222,7 @@ subprojects {
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':server',
-    "org.elasticsearch:elasticsearch-cli:${version}": ':libs:cli',
+    "org.elasticsearch:elasticsearch-cli:${version}": ':libs:elasticsearch-cli',
     "org.elasticsearch:elasticsearch-core:${version}": ':libs:core',
     "org.elasticsearch:elasticsearch-nio:${version}": ':libs:nio',
     "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
@@ -622,6 +622,21 @@ gradle.projectsEvaluated {
       }
     }
   }
+  // Having the same group and name for distinct projects causes Gradle to consider them equal when resolving
+  // dependencies leading to hard to debug failures. Run a check across all project to prevent this from happening.
+  // see: https://github.com/gradle/gradle/issues/847
+  Map coordsToProject = [:]
+  project.allprojects.forEach { p ->
+    String coords = "${p.group}:${p.name}"
+    if (false == coordsToProject.putIfAbsent(coords, p)) {
+      throw new GradleException(
+              "Detected that two projects: ${p.path} and ${coordsToProject[coords].path} " +
+                      "have the same name and group: ${coords}. " +
+                      "This doesn't currently work correctly in Gradle, see: " +
+                      "https://github.com/gradle/gradle/issues/847"
+      )
+    }
+  }
 }
 
 if (System.properties.get("build.compare") != null) {
diff --git a/client/test/build.gradle b/client/test/build.gradle
index 59c45186fe7..cc69a1828dc 100644
--- a/client/test/build.gradle
+++ b/client/test/build.gradle
@@ -25,6 +25,8 @@ apply plugin: 'elasticsearch.build'
 targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
+group = "${group}.client.test"
+
 dependencies {
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
diff --git a/libs/cli/build.gradle b/libs/cli/build.gradle
index 91fbca19eca..00d6d96ef0d 100644
--- a/libs/cli/build.gradle
+++ b/libs/cli/build.gradle
@@ -24,16 +24,6 @@ apply plugin: 'nebula.optional-base'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 
-publishing {
-    publications {
-        nebula {
-            artifactId 'elasticsearch-cli'
-        }
-    }
-}
-
-archivesBaseName = 'elasticsearch-cli'
-
 dependencies {
     compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
     compile "org.elasticsearch:elasticsearch-core:${version}"
diff --git a/plugins/discovery-ec2/qa/build.gradle b/plugins/discovery-ec2/qa/build.gradle
index e69de29bb2d..0aed6df8838 100644
--- a/plugins/discovery-ec2/qa/build.gradle
+++ b/plugins/discovery-ec2/qa/build.gradle
@@ -0,0 +1 @@
+group = "${group}.plugins.discovery-ec2.qa"
diff --git a/plugins/repository-azure/qa/build.gradle b/plugins/repository-azure/qa/build.gradle
index e69de29bb2d..5c25485a8f5 100644
--- a/plugins/repository-azure/qa/build.gradle
+++ b/plugins/repository-azure/qa/build.gradle
@@ -0,0 +1 @@
+group = "${group}.plugins.repository-azure.qa"
diff --git a/plugins/repository-gcs/qa/build.gradle b/plugins/repository-gcs/qa/build.gradle
index e69de29bb2d..d10d9050dfe 100644
--- a/plugins/repository-gcs/qa/build.gradle
+++ b/plugins/repository-gcs/qa/build.gradle
@@ -0,0 +1 @@
+group = "${group}.plugins.repository-gcs.qa"
diff --git a/settings.gradle b/settings.gradle
index 5904cc4daf4..bdd866e622b 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -131,3 +131,5 @@ if (extraProjects.exists()) {
 
 // enable in preparation for Gradle 5.0
 enableFeaturePreview('STABLE_PUBLISHING')
+
+project(":libs:cli").name = 'elasticsearch-cli'
diff --git a/x-pack/qa/rolling-upgrade/with-system-key/build.gradle b/x-pack/qa/rolling-upgrade/with-system-key/build.gradle
index e69de29bb2d..03505e01ded 100644
--- a/x-pack/qa/rolling-upgrade/with-system-key/build.gradle
+++ b/x-pack/qa/rolling-upgrade/with-system-key/build.gradle
@@ -0,0 +1 @@
+group = "${group}.x-pack.qa.rolling-upgrade.with-system-key"
diff --git a/x-pack/qa/rolling-upgrade/without-system-key/build.gradle b/x-pack/qa/rolling-upgrade/without-system-key/build.gradle
index e69de29bb2d..aa7ac502eb3 100644
--- a/x-pack/qa/rolling-upgrade/without-system-key/build.gradle
+++ b/x-pack/qa/rolling-upgrade/without-system-key/build.gradle
@@ -0,0 +1 @@
+group = "${group}.x-pack.qa.rolling-upgrade.without-system-key"
diff --git a/x-pack/qa/sql/security/build.gradle b/x-pack/qa/sql/security/build.gradle
index 5c3169d9d20..15f7734f942 100644
--- a/x-pack/qa/sql/security/build.gradle
+++ b/x-pack/qa/sql/security/build.gradle
@@ -4,6 +4,8 @@ dependencies {
 
 Project mainProject = project
 
+group = "${group}.x-pack.qa.sql.security"
+
 subprojects {
   // Use resources from the parent project in subprojects
   sourceSets {

From 644a92f1582ceeda6981c0653a5e455f5ec43e3c Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Thu, 19 Jul 2018 09:34:23 +0200
Subject: [PATCH 095/260] Fix rollup on date fields that don't support
 epoch_millis (#31890)

The rollup indexer uses a range query to select the next page
of results based on the last time bucket of the previous round
and the `delay` configured on the rollup job. This query uses
the `epoch_millis` format implicitly but doesn't set the `format`.
This result in errors during the rollup job if the field
definition doesn't allow this format. It can also miss documents
if the format is not accepted but another format in the field
definition is able to parse the query (e.g.: `epoch_second`).
This change ensures that we use `epoch_millis` as the only format
to parse the rollup range query.
---
 .../xpack/rollup/job/RollupIndexer.java       |  3 +-
 .../job/RollupIndexerIndexingTests.java       |  5 +-
 .../elasticsearch/multi_node/RollupIT.java    | 48 ++++++++++++++-----
 3 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
index 1711c0e34eb..c7d29451ab3 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
@@ -425,7 +425,8 @@ public abstract class RollupIndexer {
         assert lowerBound <= maxBoundary;
         final RangeQueryBuilder query = new RangeQueryBuilder(fieldName)
                 .gte(lowerBound)
-                .lt(maxBoundary);
+                .lt(maxBoundary)
+                .format("epoch_millis");
         return query;
     }
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
index bf4f4892ef6..f658fa574eb 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
@@ -29,6 +29,8 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchResponseSections;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.joda.DateMathParser;
+import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.rounding.Rounding;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexSettings;
@@ -506,6 +508,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase {
     private Map<String, MappedFieldType> createFieldTypes(RollupJobConfig job) {
         Map<String, MappedFieldType> fieldTypes = new HashMap<>();
         MappedFieldType fieldType = new DateFieldMapper.Builder(job.getGroupConfig().getDateHisto().getField())
+                .dateTimeFormatter(Joda.forPattern(randomFrom("basic_date", "date_optional_time", "epoch_second")))
                 .build(new Mapper.BuilderContext(settings.getSettings(), new ContentPath(0)))
                 .fieldType();
         fieldTypes.put(fieldType.name(), fieldType);
@@ -618,7 +621,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase {
             RangeQueryBuilder range = (RangeQueryBuilder) request.source().query();
             final DateTimeZone timeZone = range.timeZone() != null ? DateTimeZone.forID(range.timeZone()) : null;
             Query query = timestampField.rangeQuery(range.from(), range.to(), range.includeLower(), range.includeUpper(),
-                    null, timeZone, null, queryShardContext);
+                    null, timeZone, new DateMathParser(Joda.forPattern(range.format())), queryShardContext);
 
             // extract composite agg
             assertThat(request.source().aggregations().getAggregatorFactories().size(), equalTo(1));
diff --git a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
index b0142ae1418..43ad4dc0a45 100644
--- a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
+++ b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
@@ -6,12 +6,16 @@
 package org.elasticsearch.multi_node;
 
 import org.apache.http.HttpStatus;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -33,8 +37,8 @@ import java.time.format.DateTimeFormatter;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.isOneOf;
@@ -73,6 +77,31 @@ public class RollupIT extends ESRestTestCase {
 
     public void testBigRollup() throws Exception {
         final int numDocs = 200;
+        String dateFormat = "strict_date_optional_time";
+
+        // create the test-index index
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.startObject("mappings").startObject("_doc")
+                    .startObject("properties")
+                    .startObject("timestamp")
+                    .field("type", "date")
+                    .field("format", dateFormat)
+                    .endObject()
+                    .startObject("value")
+                    .field("type", "integer")
+                    .endObject()
+                    .endObject()
+                    .endObject().endObject();
+            }
+            builder.endObject();
+            final StringEntity entity = new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
+            Request req = new Request("PUT", "rollup-docs");
+            req.setEntity(entity);
+            client().performRequest(req);
+        }
+
 
         // index documents for the rollup job
         final StringBuilder bulk = new StringBuilder();
@@ -88,13 +117,15 @@ public class RollupIT extends ESRestTestCase {
         bulkRequest.addParameter("refresh", "true");
         bulkRequest.setJsonEntity(bulk.toString());
         client().performRequest(bulkRequest);
+
         // create the rollup job
         final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-job-test");
+        int pageSize = randomIntBetween(2, 50);
         createRollupJobRequest.setJsonEntity("{"
             + "\"index_pattern\":\"rollup-*\","
             + "\"rollup_index\":\"results-rollup\","
-            + "\"cron\":\"*/1 * * * * ?\","             // fast cron and big page size so test runs quickly
-            + "\"page_size\":20,"
+            + "\"cron\":\"*/1 * * * * ?\","             // fast cron so test runs quickly
+            + "\"page_size\":" + pageSize + ","
             + "\"groups\":{"
             + "    \"date_histogram\":{"
             + "        \"field\":\"timestamp\","
@@ -142,7 +173,8 @@ public class RollupIT extends ESRestTestCase {
             "    \"date_histo\": {\n" +
             "      \"date_histogram\": {\n" +
             "        \"field\": \"timestamp\",\n" +
-            "        \"interval\": \"1h\"\n" +
+            "        \"interval\": \"1h\",\n" +
+            "        \"format\": \"date_time\"\n" +
             "      },\n" +
             "      \"aggs\": {\n" +
             "        \"the_max\": {\n" +
@@ -226,7 +258,7 @@ public class RollupIT extends ESRestTestCase {
 
     }
 
-    private void waitForRollUpJob(final String rollupJob,String[] expectedStates) throws Exception {
+    private void waitForRollUpJob(final String rollupJob, String[] expectedStates) throws Exception {
         assertBusy(() -> {
             final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/" + rollupJob);
             Response getRollupJobResponse = client().performRequest(getRollupJobRequest);
@@ -317,10 +349,4 @@ public class RollupIT extends ESRestTestCase {
             }
         }
     }
-
-    private static String responseEntityToString(Response response) throws Exception {
-        try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
-            return reader.lines().collect(Collectors.joining("\n"));
-        }
-    }
 }

From f232c36c197d7d2e470823e672f66e85e4e1e20c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Thu, 19 Jul 2018 09:41:32 +0200
Subject: [PATCH 096/260] Fix comments causing errors with Java 11

---
 .../aggregations/bucket/nested/NestedAggregatorTests.java       | 2 +-
 .../bucket/nested/ReverseNestedAggregatorTests.java             | 2 +-
 .../bucket/significant/SignificantTermsAggregatorTests.java     | 2 +-
 .../bucket/significant/SignificantTextAggregatorTests.java      | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
index 4555809bad9..c893e595964 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
@@ -91,7 +91,7 @@ public class NestedAggregatorTests extends AggregatorTestCase {
     private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
 
     /**
-     * For each provided field type, we also register an alias with name <field>-alias.
+     * For each provided field type, we also register an alias with name {@code <field>-alias}.
      */
     @Override
     protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
index bc870bf4dca..99322af2264 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
@@ -59,7 +59,7 @@ public class ReverseNestedAggregatorTests extends AggregatorTestCase {
     private static final String MAX_AGG_NAME = "maxAgg";
 
     /**
-     * For each provided field type, we also register an alias with name <field>-alias.
+     * For each provided field type, we also register an alias with name {@code <field>-alias}.
      */
     @Override
     protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
index 70f9667ce7b..0485d4f5855 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorTests.java
@@ -77,7 +77,7 @@ public class SignificantTermsAggregatorTests extends AggregatorTestCase {
     }
 
     /**
-     * For each provided field type, we also register an alias with name <field>-alias.
+     * For each provided field type, we also register an alias with name {@code <field>-alias}.
      */
     @Override
     protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
index c63d5cb7d39..dbff6daed62 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorTests.java
@@ -53,7 +53,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.signific
 public class SignificantTextAggregatorTests extends AggregatorTestCase {
 
     /**
-     * For each provided field type, we also register an alias with name <field>-alias.
+     * For each provided field type, we also register an alias with name {@code <field>-alias}.
      */
     @Override
     protected Map<String, MappedFieldType> getFieldAliases(MappedFieldType... fieldTypes) {

From 185689ad75955522872acfc7cf21170036680fea Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 19 Jul 2018 01:24:49 -0700
Subject: [PATCH 097/260] Build: Remove pom generation for plugin zip files
 (#32180)

In 1.x and 2.x, plugins were published to maven and the plugin
installer downloaded them from there. This was later changed to install
from the download service, and in 5.0 plugin zips were no longer
published to maven. However, the build still currently produces an
unused pom file. This is troublesome in the special case when the main
jar of a plugin needs to be published (and thus needs a pom file of
the same name).

closes #31946
---
 .../gradle/plugin/PluginBuildPlugin.groovy    | 36 ++++---------------
 1 file changed, 7 insertions(+), 29 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index d76084bf22e..7f6f337e8a9 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -75,10 +75,10 @@ public class PluginBuildPlugin extends BuildPlugin {
                 // and generate a different pom for the zip
                 addClientJarPomGeneration(project)
                 addClientJarTask(project)
-            } else {
-                // no client plugin, so use the pom file from nebula, without jar, for the zip
-                project.ext.set("nebulaPublish.maven.jar", false)
             }
+            // while the jar isn't normally published, we still at least build a pom of deps
+            // in case it is published, for instance when other plugins extend this plugin
+            configureJarPom(project)
 
             project.integTestCluster.dependsOn(project.bundlePlugin)
             project.tasks.run.dependsOn(project.bundlePlugin)
@@ -94,7 +94,6 @@ public class PluginBuildPlugin extends BuildPlugin {
             }
 
             if (isModule == false || isXPackModule) {
-                addZipPomGeneration(project)
                 addNoticeGeneration(project)
             }
 
@@ -239,36 +238,15 @@ public class PluginBuildPlugin extends BuildPlugin {
         }
     }
 
-    /** Adds a task to generate a pom file for the zip distribution. */
-    public static void addZipPomGeneration(Project project) {
+    /** Configure the pom for the main jar of this plugin */
+    protected static void configureJarPom(Project project) {
         project.plugins.apply(ScmInfoPlugin.class)
         project.plugins.apply(MavenPublishPlugin.class)
 
         project.publishing {
             publications {
-                zip(MavenPublication) {
-                    artifact project.bundlePlugin
-                }
-                /* HUGE HACK: the underlying maven publication library refuses to deploy any attached artifacts
-                 * when the packaging type is set to 'pom'. But Sonatype's OSS repositories require source files
-                 * for artifacts that are of type 'zip'. We already publish the source and javadoc for Elasticsearch
-                 * under the various other subprojects. So here we create another publication using the same
-                 * name that has the "real" pom, and rely on the fact that gradle will execute the publish tasks
-                 * in alphabetical order. This lets us publish the zip file and even though the pom says the
-                 * type is 'pom' instead of 'zip'. We cannot setup a dependency between the tasks because the
-                 * publishing tasks are created *extremely* late in the configuration phase, so that we cannot get
-                 * ahold of the actual task. Furthermore, this entire hack only exists so we can make publishing to
-                 * maven local work, since we publish to maven central externally. */
-                zipReal(MavenPublication) {
-                    artifactId = project.pluginProperties.extension.name
-                    pom.withXml { XmlProvider xml ->
-                        Node root = xml.asNode()
-                        root.appendNode('name', project.pluginProperties.extension.name)
-                        root.appendNode('description', project.pluginProperties.extension.description)
-                        root.appendNode('url', urlFromOrigin(project.scminfo.origin))
-                        Node scmNode = root.appendNode('scm')
-                        scmNode.appendNode('url', project.scminfo.origin)
-                    }
+                nebula(MavenPublication) {
+                    artifactId project.pluginProperties.extension.name
                 }
             }
         }

From 202894b8322571c447bc20a6262fd49da89f9bf5 Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Thu, 19 Jul 2018 10:40:54 +0200
Subject: [PATCH 098/260] Rest HL client: Add put watch action (#32026)

Relates #29827

This implementation behaves like the current transport client, that you basically cannot configure a Watch POJO representation as an argument to the put watch API, but only a bytes reference. You can use the the `WatchSourceBuilder` from the `org.elasticsearch.plugin:x-pack-core` dependency to build watches.

This commit also changes the license type to trial, so that watcher is available in high level rest client tests.

/cc @hub-cap
---
 client/rest-high-level/build.gradle           |   4 +
 .../client/RequestConverters.java             |  20 +++
 .../elasticsearch/client/WatcherClient.java   |  64 +++++++++
 .../org/elasticsearch/client/XPackClient.java |   7 +
 .../elasticsearch/client/PingAndInfoIT.java   |   8 +-
 .../client/RequestConvertersTests.java        |  33 ++++-
 .../client/RestHighLevelClientTests.java      |   4 +-
 .../org/elasticsearch/client/WatcherIT.java   |  46 +++++++
 .../MiscellaneousDocumentationIT.java         |   5 +-
 .../documentation/WatcherDocumentationIT.java |  92 +++++++++++++
 .../high-level/supported-apis.asciidoc        |   3 +-
 .../x-pack/watcher/put-watch.asciidoc         |  55 ++++++++
 .../x-pack-info.asciidoc                      |   0
 .../x-pack-usage.asciidoc                     |   0
 .../core/watcher/client/WatcherClient.java    |   4 +-
 .../transport/actions/put/PutWatchAction.java |   1 +
 .../actions/put/PutWatchRequestBuilder.java   |   4 +-
 .../actions/put/PutWatchResponse.java         |  59 ---------
 .../exporter/local/LocalExporter.java         |   4 +-
 .../rest/action/RestPutWatchAction.java       |   4 +-
 .../actions/put/TransportPutWatchAction.java  |   4 +-
 .../actions/ActionErrorIntegrationTests.java  |   2 +-
 .../actions/TimeThrottleIntegrationTests.java |   2 +-
 .../throttler/ActionThrottleTests.java        |   2 +-
 .../history/HistoryActionConditionTests.java  |   2 +-
 .../HistoryTemplateEmailMappingsTests.java    |   2 +-
 .../HistoryTemplateHttpMappingsTests.java     |   2 +-
 ...storyTemplateIndexActionMappingsTests.java |   2 +-
 ...storyTemplateSearchInputMappingsTests.java |   2 +-
 .../HistoryTemplateTimeMappingsTests.java     |   2 +-
 .../WatcherExecutorServiceBenchmark.java      |   2 +-
 .../test/integration/BasicWatcherTests.java   |   2 +-
 .../ExecutionVarsIntegrationTests.java        |   2 +-
 .../integration/HistoryIntegrationTests.java  |   2 +-
 .../test/integration/WatchAckTests.java       |   2 +-
 .../transform/TransformIntegrationTests.java  |   2 +-
 .../action/WatchRequestValidationTests.java   |   2 +-
 .../action/activate/ActivateWatchTests.java   |   4 +-
 .../action/delete/DeleteWatchTests.java       |   2 +-
 .../action/execute/ExecuteWatchTests.java     |   2 +-
 .../transport/action/get/GetWatchTests.java   |   2 +-
 .../put/PutWatchSerializationTests.java       |   2 +-
 .../put/TransportPutWatchActionTests.java     |   4 +-
 .../xpack/watcher}/PutWatchRequest.java       |  56 ++++----
 .../xpack/watcher/PutWatchResponse.java       | 124 ++++++++++++++++++
 .../xpack/watcher/PutWatchResponseTests.java  |  45 +++++++
 46 files changed, 570 insertions(+), 124 deletions(-)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
 create mode 100644 docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc
 rename docs/java-rest/high-level/{miscellaneous => x-pack}/x-pack-info.asciidoc (100%)
 rename docs/java-rest/high-level/{miscellaneous => x-pack}/x-pack-usage.asciidoc (100%)
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchResponse.java
 rename x-pack/{plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put => protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher}/PutWatchRequest.java (73%)
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java
 create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java

diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index a1260894bf7..65c5d094c71 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -70,3 +70,7 @@ forbiddenApisMain {
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
   signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
 }
+
+integTestCluster {
+  setting 'xpack.license.self_generated.type', 'trial'
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index 9dbd4916c77..a6122b0681e 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -106,6 +106,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
@@ -1097,6 +1098,25 @@ final class RequestConverters {
         return request;
     }
 
+    static Request xPackWatcherPutWatch(PutWatchRequest putWatchRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("watcher")
+            .addPathPartAsIs("watch")
+            .addPathPart(putWatchRequest.getId())
+            .build();
+
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        Params params = new Params(request).withVersion(putWatchRequest.getVersion());
+        if (putWatchRequest.isActive() == false) {
+            params.putParam("active", "false");
+        }
+        ContentType contentType = createContentType(putWatchRequest.xContentType());
+        BytesReference source = putWatchRequest.getSource();
+        request.setEntity(new ByteArrayEntity(source.toBytesRef().bytes, 0, source.length(), contentType));
+        return request;
+    }
+
     static Request xpackUsage(XPackUsageRequest usageRequest) {
         Request request = new Request(HttpGet.METHOD_NAME, "/_xpack/usage");
         Params parameters = new Params(request);
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
new file mode 100644
index 00000000000..73c92ba5c45
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptySet;
+
+public final class WatcherClient {
+
+    private final RestHighLevelClient restHighLevelClient;
+
+    WatcherClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Put a watch into the cluster
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-put-watch.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public PutWatchResponse putWatch(PutWatchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::xPackWatcherPutWatch, options,
+            PutWatchResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously put a watch into the cluster
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-put-watch.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void putWatchAsync(PutWatchRequest request, RequestOptions options,
+                              ActionListener<PutWatchResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xPackWatcherPutWatch, options,
+            PutWatchResponse::fromXContent, listener, emptySet());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
index a497619b987..4acaadfdb85 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
@@ -39,10 +39,17 @@ import static java.util.Collections.emptySet;
  * X-Pack APIs on elastic.co</a> for more information.
  */
 public final class XPackClient {
+
     private final RestHighLevelClient restHighLevelClient;
+    private final WatcherClient watcherClient;
 
     XPackClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
+        this.watcherClient = new WatcherClient(restHighLevelClient);
+    }
+
+    public WatcherClient watcher() {
+        return watcherClient;
     }
 
     /**
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java
index b45f52f9e44..5f38316fd75 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java
@@ -66,13 +66,13 @@ public class PingAndInfoIT extends ESRestHighLevelClientTestCase {
 
         assertEquals(mainResponse.getBuild().shortHash(), info.getBuildInfo().getHash());
 
-        assertEquals("basic", info.getLicenseInfo().getType());
-        assertEquals("basic", info.getLicenseInfo().getMode());
+        assertEquals("trial", info.getLicenseInfo().getType());
+        assertEquals("trial", info.getLicenseInfo().getMode());
         assertEquals(LicenseStatus.ACTIVE, info.getLicenseInfo().getStatus());
 
         FeatureSet graph = info.getFeatureSetsInfo().getFeatureSets().get("graph");
         assertNotNull(graph.description());
-        assertFalse(graph.available());
+        assertTrue(graph.available());
         assertTrue(graph.enabled());
         assertNull(graph.nativeCodeInfo());
         FeatureSet monitoring = info.getFeatureSetsInfo().getFeatureSets().get("monitoring");
@@ -82,7 +82,7 @@ public class PingAndInfoIT extends ESRestHighLevelClientTestCase {
         assertNull(monitoring.nativeCodeInfo());
         FeatureSet ml = info.getFeatureSetsInfo().getFeatureSets().get("ml");
         assertNotNull(ml.description());
-        assertFalse(ml.available());
+        assertTrue(ml.available());
         assertTrue(ml.enabled());
         assertEquals(mainResponse.getVersion().toString(),
                 ml.nativeCodeInfo().get("version").toString().replace("-SNAPSHOT", ""));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index fb4e3b22712..c1f47feb33d 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -41,9 +41,9 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequ
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
@@ -125,6 +125,7 @@ import org.elasticsearch.index.rankeval.RankEvalSpec;
 import org.elasticsearch.index.rankeval.RatedRequest;
 import org.elasticsearch.index.rankeval.RestRankEvalAction;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.ScriptType;
@@ -145,6 +146,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.RandomObjects;
 import org.hamcrest.CoreMatchers;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
@@ -2523,6 +2525,35 @@ public class RequestConvertersTests extends ESTestCase {
         assertEquals(expectedParams, request.getParameters());
     }
 
+    public void testXPackPutWatch() throws Exception {
+        PutWatchRequest putWatchRequest = new PutWatchRequest();
+        String watchId = randomAlphaOfLength(10);
+        putWatchRequest.setId(watchId);
+        String body = randomAlphaOfLength(20);
+        putWatchRequest.setSource(new BytesArray(body), XContentType.JSON);
+
+        Map<String, String> expectedParams = new HashMap<>();
+        if (randomBoolean()) {
+            putWatchRequest.setActive(false);
+            expectedParams.put("active", "false");
+        }
+
+        if (randomBoolean()) {
+            long version = randomLongBetween(10, 100);
+            putWatchRequest.setVersion(version);
+            expectedParams.put("version", String.valueOf(version));
+        }
+
+        Request request = RequestConverters.xPackWatcherPutWatch(putWatchRequest);
+        assertEquals(HttpPut.METHOD_NAME, request.getMethod());
+        assertEquals("/_xpack/watcher/watch/" + watchId, request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertThat(request.getEntity().getContentType().getValue(), is(XContentType.JSON.mediaTypeWithoutParameters()));
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        request.getEntity().writeTo(bos);
+        assertThat(bos.toString("UTF-8"), is(body));
+    }
+
     /**
      * Randomize the {@link FetchSourceContext} request parameters.
      */
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 47870125aa2..5acc6f5552f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -767,7 +767,9 @@ public class RestHighLevelClientTests extends ESTestCase {
 
     private static Stream<Tuple<String, Method>> getSubClientMethods(String namespace, Class<?> clientClass) {
         return Arrays.stream(clientClass.getMethods()).filter(method -> method.getDeclaringClass().equals(clientClass))
-                .map(method -> Tuple.tuple(namespace + "." + toSnakeCase(method.getName()), method));
+                .map(method -> Tuple.tuple(namespace + "." + toSnakeCase(method.getName()), method))
+                .flatMap(tuple -> tuple.v2().getReturnType().getName().endsWith("Client")
+                    ? getSubClientMethods(tuple.v1(), tuple.v2().getReturnType()) : Stream.of(tuple));
     }
 
     private static String toSnakeCase(String camelCase) {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
new file mode 100644
index 00000000000..dec438a47ab
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
+
+import static org.hamcrest.Matchers.is;
+
+public class WatcherIT extends ESRestHighLevelClientTestCase {
+
+    public void testPutWatch() throws Exception {
+        String watchId = randomAlphaOfLength(10);
+        String json = "{ \n" +
+            "  \"trigger\": { \"schedule\": { \"interval\": \"10h\" } },\n" +
+            "  \"input\": { \"none\": {} },\n" +
+            "  \"actions\": { \"logme\": { \"logging\": { \"text\": \"{{ctx.payload}}\" } } }\n" +
+            "}";
+        BytesReference bytesReference = new BytesArray(json);
+        PutWatchRequest putWatchRequest = new PutWatchRequest(watchId, bytesReference, XContentType.JSON);
+        PutWatchResponse putWatchResponse = highLevelClient().xpack().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT);
+        assertThat(putWatchResponse.isCreated(), is(true));
+        assertThat(putWatchResponse.getId(), is(watchId));
+        assertThat(putWatchResponse.getVersion(), is(1L));
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java
index a99b991620a..a9fe4aba2f7 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java
@@ -39,11 +39,13 @@ import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.protocol.xpack.XPackUsageResponse;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.EnumSet;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 
 /**
@@ -97,8 +99,7 @@ public class MiscellaneousDocumentationIT extends ESRestHighLevelClientTestCase
             //tag::x-pack-info-response
             BuildInfo build = response.getBuildInfo();                 // <1>
             LicenseInfo license = response.getLicenseInfo();           // <2>
-            assertEquals(XPackInfoResponse.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS,
-                    license.getExpiryDate());                          // <3>
+            assertThat(license.getExpiryDate(), is(greaterThan(Instant.now().toEpochMilli())));  // <3>
             FeatureSetsInfo features = response.getFeatureSetsInfo();  // <4>
             //end::x-pack-info-response
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
new file mode 100644
index 00000000000..df51d896cda
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.LatchedActionListener;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    public void testPutWatch() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        {
+            //tag::x-pack-put-watch-execute
+            // you can also use the WatchSourceBuilder from org.elasticsearch.plugin:x-pack-core to create a watch programmatically
+            BytesReference watch = new BytesArray("{ \n" +
+                "  \"trigger\": { \"schedule\": { \"interval\": \"10h\" } },\n" +
+                "  \"input\": { \"simple\": { \"foo\" : \"bar\" } },\n" +
+                "  \"actions\": { \"logme\": { \"logging\": { \"text\": \"{{ctx.payload}}\" } } }\n" +
+                "}");
+            PutWatchRequest request = new PutWatchRequest("my_watch_id", watch, XContentType.JSON);
+            request.setActive(false); // <1>
+            PutWatchResponse response = client.xpack().watcher().putWatch(request, RequestOptions.DEFAULT);
+            //end::x-pack-put-watch-execute
+
+            //tag::x-pack-put-watch-response
+            String watchId = response.getId(); // <1>
+            boolean isCreated = response.isCreated(); // <2>
+            long version = response.getVersion(); // <3>
+            //end::x-pack-put-watch-response
+        }
+
+        {
+            BytesReference watch = new BytesArray("{ \n" +
+                "  \"trigger\": { \"schedule\": { \"interval\": \"10h\" } },\n" +
+                "  \"input\": { \"simple\": { \"foo\" : \"bar\" } },\n" +
+                "  \"actions\": { \"logme\": { \"logging\": { \"text\": \"{{ctx.payload}}\" } } }\n" +
+                "}");
+            PutWatchRequest request = new PutWatchRequest("my_other_watch_id", watch, XContentType.JSON);
+            // tag::x-pack-put-watch-execute-listener
+            ActionListener<PutWatchResponse> listener = new ActionListener<PutWatchResponse>() {
+                @Override
+                public void onResponse(PutWatchResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::x-pack-put-watch-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::x-pack-put-watch-execute-async
+            client.xpack().watcher().putWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::x-pack-put-watch-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+}
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index cf38040e865..d952870677b 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -57,7 +57,8 @@ The Java High Level REST Client supports the following Miscellaneous APIs:
 
 include::miscellaneous/main.asciidoc[]
 include::miscellaneous/ping.asciidoc[]
-include::miscellaneous/x-pack-info.asciidoc[]
+include::x-pack/x-pack-info.asciidoc[]
+include::x-pack/watcher/put-watch.asciidoc[]
 
 == Indices APIs
 
diff --git a/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc b/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc
new file mode 100644
index 00000000000..c803c54eb5e
--- /dev/null
+++ b/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc
@@ -0,0 +1,55 @@
+[[java-rest-high-x-pack-watcher-put-watch]]
+=== X-Pack Info API
+
+[[java-rest-high-x-pack-watcher-put-watch-execution]]
+==== Execution
+
+General information about the installed {watcher} features can be retrieved
+using the `watcher()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-execute]
+--------------------------------------------------
+<1> Allows to store the watch, but to not trigger it. Defaults to `true`
+
+[[java-rest-high-x-pack-watcher-put-watch-response]]
+==== Response
+
+The returned `XPackPutWatchResponse` contain `created`, `id`,
+and `version` information.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-response]
+--------------------------------------------------
+<1> `_id` contains id of the watch
+<2> `created` is a boolean indicating whether the watch was created for the first time
+<3> `_version` returns the newly created version
+
+[[java-rest-high-x-pack-watcher-put-watch-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-execute-async]
+--------------------------------------------------
+<1> The `XPackPutWatchRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `XPackPutWatchResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
diff --git a/docs/java-rest/high-level/miscellaneous/x-pack-info.asciidoc b/docs/java-rest/high-level/x-pack/x-pack-info.asciidoc
similarity index 100%
rename from docs/java-rest/high-level/miscellaneous/x-pack-info.asciidoc
rename to docs/java-rest/high-level/x-pack/x-pack-info.asciidoc
diff --git a/docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc b/docs/java-rest/high-level/x-pack/x-pack-usage.asciidoc
similarity index 100%
rename from docs/java-rest/high-level/miscellaneous/x-pack-usage.asciidoc
rename to docs/java-rest/high-level/x-pack/x-pack-usage.asciidoc
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
index 10c4f0fffc3..063f1f655a4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
@@ -8,6 +8,8 @@ package org.elasticsearch.xpack.core.watcher.client;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequestBuilder;
@@ -29,9 +31,7 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchReques
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequestBuilder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchAction.java
index 56cedc457bd..509116b018e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchAction.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.watcher.transport.actions.put;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 
 /**
  * This action puts an watch into the watch index and adds it to the scheduler
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchRequestBuilder.java
index 050ac38dbb0..840ff560fba 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchRequestBuilder.java
@@ -9,6 +9,8 @@ import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 
 public class PutWatchRequestBuilder extends ActionRequestBuilder<PutWatchRequest, PutWatchResponse> {
@@ -43,7 +45,7 @@ public class PutWatchRequestBuilder extends ActionRequestBuilder<PutWatchRequest
      * @param source the source of the watch to be created
      */
     public PutWatchRequestBuilder setSource(WatchSourceBuilder source) {
-        request.setSource(source);
+        request.setSource(source.buildAsBytes(XContentType.JSON), XContentType.JSON);
         return this;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchResponse.java
deleted file mode 100644
index b84901ecac1..00000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchResponse.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.watcher.transport.actions.put;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-
-/**
- * The Response for a put watch action
- */
-public class PutWatchResponse extends ActionResponse {
-
-    private String id;
-    private long version;
-    private boolean created;
-
-    public PutWatchResponse() {
-    }
-
-    public PutWatchResponse(String id, long version, boolean created) {
-        this.id = id;
-        this.version = version;
-        this.created = created;
-    }
-
-    public String getId() {
-        return id;
-    }
-
-    public long getVersion() {
-        return version;
-    }
-
-    public boolean isCreated() {
-        return created;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeString(id);
-        out.writeVLong(version);
-        out.writeBoolean(created);
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        id = in.readString();
-        version = in.readVLong();
-        created = in.readBoolean();
-    }
-}
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
index 93c74ddf3aa..08618b45cf5 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
@@ -39,6 +39,8 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
@@ -47,8 +49,6 @@ import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
index 0386ca47c34..6fe93aec3a4 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
@@ -9,6 +9,8 @@ import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -17,8 +19,6 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.security.rest.RestRequestFilter;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchAction.java
index 5be1236574f..cbee4c7dc94 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchAction.java
@@ -19,13 +19,13 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.transport.actions.WatcherTransportAction;
 import org.elasticsearch.xpack.watcher.watch.WatchParser;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionErrorIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionErrorIntegrationTests.java
index 144a0a75b15..6590d6c271b 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionErrorIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionErrorIntegrationTests.java
@@ -8,9 +8,9 @@ package org.elasticsearch.xpack.watcher.actions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.actions.index.IndexAction;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
index c0be6c1c17f..9b851131d7d 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
@@ -8,13 +8,13 @@ package org.elasticsearch.xpack.watcher.actions;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
 import java.util.Map;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
index 05256ba5fc4..b2a1e7bb2cd 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
@@ -10,6 +10,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.xpack.core.watcher.actions.Action;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
@@ -17,7 +18,6 @@ import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.actions.email.EmailAction;
 import org.elasticsearch.xpack.watcher.actions.index.IndexAction;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
index 189bf1d5b05..9093a6f86ae 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.watcher.history;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -18,7 +19,6 @@ import org.elasticsearch.xpack.core.watcher.condition.Condition;
 import org.elasticsearch.xpack.core.watcher.condition.ExecutableCondition;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.input.Input;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.condition.NeverCondition;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
index 38008194b0a..2f578a1b880 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
@@ -7,12 +7,12 @@ package org.elasticsearch.xpack.watcher.history;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.notification.email.EmailTemplate;
 import org.elasticsearch.xpack.watcher.notification.email.support.EmailServer;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
index 51652078dc9..0e120793fbc 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
@@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.test.http.MockResponse;
@@ -18,7 +19,6 @@ import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
index 6bf273b194c..e7b6ad92231 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
@@ -6,11 +6,11 @@
 package org.elasticsearch.xpack.watcher.history;
 
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
index 17575f58ec2..69de3bcf871 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
@@ -8,11 +8,11 @@ package org.elasticsearch.xpack.watcher.history;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
index 2259ad87eb1..0ad674f7ed2 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
@@ -10,9 +10,9 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
index 9ae83c79e95..4ca412c5ce0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
@@ -11,12 +11,12 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
 import org.elasticsearch.xpack.watcher.Watcher;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.condition.ScriptCondition;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
index 1c1d8dde8cf..18f151b4f6a 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
@@ -11,6 +11,7 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -20,7 +21,6 @@ import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
index 2f69cc95a50..0b3c0fc28ec 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
@@ -8,12 +8,12 @@ package org.elasticsearch.xpack.watcher.test.integration;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.condition.ScriptCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.hamcrest.Matcher;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
index de48c240a3a..5c9dafeaca0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
@@ -10,6 +10,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -17,7 +18,6 @@ import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.input.Input;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
index d98d6a44daf..1ae49265352 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
@@ -14,6 +14,7 @@ import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
@@ -23,7 +24,6 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchReques
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
index 1f7335aef0d..d629c54934f 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
@@ -11,10 +11,10 @@ import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
index b1fa736292c..64b913f96a0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
@@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequest;
@@ -16,7 +17,6 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateW
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
 
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java
index 5da3f05177a..1080880ea8b 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java
@@ -15,13 +15,13 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
@@ -47,7 +47,7 @@ public class ActivateWatchTests extends AbstractWatcherIntegrationTestCase {
     protected boolean timeWarped() {
         return false;
     }
-    
+
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30699")
     public void testDeactivateAndActivate() throws Exception {
         PutWatchResponse putWatchResponse = watcherClient().preparePutWatch()
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
index b50204abf41..76f71b9c95e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.watcher.transport.action.delete;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
@@ -15,7 +16,6 @@ import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/execute/ExecuteWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/execute/ExecuteWatchTests.java
index c307764c29f..53d5458706a 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/execute/ExecuteWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/execute/ExecuteWatchTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.watcher.transport.action.execute;
 
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
@@ -14,7 +15,6 @@ import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java
index 9d13a2ec2a1..226879b5e30 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java
@@ -7,11 +7,11 @@ package org.elasticsearch.xpack.watcher.transport.action.get;
 
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/put/PutWatchSerializationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/put/PutWatchSerializationTests.java
index a26022d97ae..df3a324bdac 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/put/PutWatchSerializationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/put/PutWatchSerializationTests.java
@@ -11,8 +11,8 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
 
 import static org.hamcrest.Matchers.is;
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java
index 6dcd5f199d5..329d9d976a9 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java
@@ -15,11 +15,11 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequest;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.test.WatchExecutionContextMockBuilder;
@@ -92,4 +92,4 @@ public class TransportPutWatchActionTests extends ESTestCase {
         assertThat(capturedHeaders.keySet(), hasSize(1));
         assertThat(capturedHeaders, hasKey(headerName));
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java
similarity index 73%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchRequest.java
rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java
index 47bc800cec7..0bfa7dc7d34 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchRequest.java
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java
@@ -1,37 +1,54 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
-package org.elasticsearch.xpack.core.watcher.transport.actions.put;
-
+package org.elasticsearch.protocol.xpack.watcher;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ValidateActions;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
-import org.elasticsearch.xpack.core.watcher.support.WatcherUtils;
 
 import java.io.IOException;
+import java.util.regex.Pattern;
 
 /**
  * This request class contains the data needed to create a watch along with the name of the watch.
  * The name of the watch will become the ID of the indexed document.
  */
-public class PutWatchRequest extends ActionRequest {
+public final class PutWatchRequest extends ActionRequest {
+
+    private static final Pattern NO_WS_PATTERN = Pattern.compile("\\S+");
 
     private String id;
     private BytesReference source;
-    private boolean active = true;
     private XContentType xContentType = XContentType.JSON;
+    private boolean active = true;
     private long version = Versions.MATCH_ANY;
 
-    public PutWatchRequest() {
+    public PutWatchRequest() {}
+
+    public PutWatchRequest(StreamInput in) throws IOException {
+        readFrom(in);
     }
 
     public PutWatchRequest(String id, BytesReference source, XContentType xContentType) {
@@ -40,8 +57,9 @@ public class PutWatchRequest extends ActionRequest {
         this.xContentType = xContentType;
     }
 
-    public PutWatchRequest(StreamInput in) throws IOException {
-        super(in);
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
         id = in.readString();
         source = in.readBytesReference();
         active = in.readBoolean();
@@ -80,13 +98,6 @@ public class PutWatchRequest extends ActionRequest {
         return source;
     }
 
-    /**
-     * Set the source of the watch
-     */
-    public void setSource(WatchSourceBuilder source) {
-        setSource(source.buildAsBytes(XContentType.JSON), XContentType.JSON);
-    }
-
     /**
      * Set the source of the watch
      */
@@ -129,7 +140,7 @@ public class PutWatchRequest extends ActionRequest {
         ActionRequestValidationException validationException = null;
         if (id == null) {
             validationException = ValidateActions.addValidationError("watch id is missing", validationException);
-        } else if (WatcherUtils.isValidId(id) == false) {
+        } else if (isValidId(id) == false) {
             validationException = ValidateActions.addValidationError("watch id contains whitespace", validationException);
         }
         if (source == null) {
@@ -141,8 +152,7 @@ public class PutWatchRequest extends ActionRequest {
         return validationException;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    public static boolean isValidId(String id) {
+        return Strings.isEmpty(id) == false && NO_WS_PATTERN.matcher(id).matches();
     }
 }
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java
new file mode 100644
index 00000000000..98467f32dd8
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponse.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.watcher;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutWatchResponse extends ActionResponse implements ToXContentObject {
+
+    private static final ObjectParser<PutWatchResponse, Void> PARSER
+        = new ObjectParser<>("x_pack_put_watch_response", PutWatchResponse::new);
+    static {
+            PARSER.declareString(PutWatchResponse::setId, new ParseField("_id"));
+            PARSER.declareLong(PutWatchResponse::setVersion, new ParseField("_version"));
+            PARSER.declareBoolean(PutWatchResponse::setCreated, new ParseField("created"));
+    }
+
+    private String id;
+    private long version;
+    private boolean created;
+
+    public PutWatchResponse() {
+    }
+
+    public PutWatchResponse(String id, long version, boolean created) {
+        this.id = id;
+        this.version = version;
+        this.created = created;
+    }
+
+    private void setId(String id) {
+        this.id = id;
+    }
+
+    private void setVersion(long version) {
+        this.version = version;
+    }
+
+    private void setCreated(boolean created) {
+        this.created = created;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    public boolean isCreated() {
+        return created;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        PutWatchResponse that = (PutWatchResponse) o;
+
+        return Objects.equals(id, that.id) && Objects.equals(version, that.version) && Objects.equals(created, that.created);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, version, created);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(id);
+        out.writeVLong(version);
+        out.writeBoolean(created);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        id = in.readString();
+        version = in.readVLong();
+        created = in.readBoolean();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject()
+            .field("_id", id)
+            .field("_version", version)
+            .field("created", created)
+            .endObject();
+    }
+
+    public static PutWatchResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java
new file mode 100644
index 00000000000..d0aadef1611
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.watcher;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class PutWatchResponseTests extends AbstractXContentTestCase<PutWatchResponse> {
+
+    @Override
+    protected PutWatchResponse createTestInstance() {
+        String id = randomAlphaOfLength(10);
+        long version = randomLongBetween(1, 10);
+        boolean created = randomBoolean();
+        return new PutWatchResponse(id, version, created);
+    }
+
+    @Override
+    protected PutWatchResponse doParseInstance(XContentParser parser) throws IOException {
+        return PutWatchResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}

From 9aa562d3dd943103e6ae5dbfb2305778e572c76a Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 19 Jul 2018 01:53:08 -0700
Subject: [PATCH 099/260] Make x-pack-core generate a pom file

This was a forgotten part of #32180
---
 x-pack/plugin/core/build.gradle | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index 5db149bc677..ca926fa0d54 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -6,6 +6,8 @@ import java.nio.file.Paths
 import java.nio.file.StandardCopyOption
 
 apply plugin: 'elasticsearch.esplugin'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-scm'
 
 archivesBaseName = 'x-pack-core'
 

From f7da55aa3c878a845795ece9ae21327801a4204c Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Thu, 19 Jul 2018 10:44:32 +0200
Subject: [PATCH 100/260] Rest test - allow for snapshots to take 0
 milliseconds

Sometimes we get lucky and things are fast :)
---
 .../resources/rest-api-spec/test/snapshot.status/10_basic.yml  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml
index c9dd85b11de..c35f2419bdc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml
@@ -38,7 +38,8 @@ setup:
   - gt: { snapshots.0.stats.total.file_count: 0 }
   - gt: { snapshots.0.stats.total.size_in_bytes: 0 }
   - is_true: snapshots.0.stats.start_time_in_millis
-  - is_true: snapshots.0.stats.time_in_millis
+## fast in memory snapshots can take less than one millisecond to complete.
+  - gte: { snapshots.0.stats.time_in_millis: 0 }
 
 ---
 "Get missing snapshot status throws an exception":

From f6d7854f76216247ed70ecf2dabb4d6edd978fff Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 19 Jul 2018 11:33:46 +0100
Subject: [PATCH 101/260] Remove indication of future multi-homing support
 (#32187)

We do not support intra-cluster connections on multiple interfaces, but the
documentation indicates that we will in future. In fact there is currently no
plan to support this, so the forward-looking documentation is misleading. This
commit

- removes the misleading sentence
- fixes that a transport profile affects outbound connections, not inbound ones
- tidies up some nearby text
---
 docs/reference/modules/transport.asciidoc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/docs/reference/modules/transport.asciidoc b/docs/reference/modules/transport.asciidoc
index 046d82cc507..257181f70c5 100644
--- a/docs/reference/modules/transport.asciidoc
+++ b/docs/reference/modules/transport.asciidoc
@@ -59,7 +59,8 @@ It also uses the common
 [float]
 ==== TCP Transport Profiles
 
-Elasticsearch allows you to bind to multiple ports on different interfaces by the use of transport profiles. See this example configuration
+Elasticsearch allows you to bind to multiple ports on different interfaces by
+the use of transport profiles. See this example configuration
 
 [source,yaml]
 --------------
@@ -71,10 +72,12 @@ transport.profiles.dmz.port: 9700-9800
 transport.profiles.dmz.bind_host: 172.16.1.2
 --------------
 
-The `default` profile is a special. It is used as fallback for any other profiles, if those do not have a specific configuration setting set.
-Note that the default profile is how other nodes in the cluster will connect to this node usually. In the future this feature will allow to enable node-to-node communication via multiple interfaces.
+The `default` profile is special. It is used as a fallback for any other
+profiles, if those do not have a specific configuration setting set, and is how
+this node connects to other nodes in the cluster.
 
-The following parameters can be configured like that
+The following parameters can be configured on each transport profile, as in the
+example above:
 
 * `port`: The port to bind to
 * `bind_host`: The host to bind

From 7c0fc209bf78e4824ca1f232b84a1dab22bc2dfa Mon Sep 17 00:00:00 2001
From: Vladimir Dolzhenko <vladimir.dolzhenko@gmail.com>
Date: Thu, 19 Jul 2018 12:54:38 +0200
Subject: [PATCH 102/260] ECS Task IAM profile credentials ignored in
 repository-s3 plugin (#31864)

ECS Task IAM profile credentials ignored in repository-s3 plugin (#31864)

Closes #26913
---
 .../gradle/test/ClusterConfiguration.groovy   |   7 +
 .../elasticsearch/gradle/test/NodeInfo.groovy |   1 +
 docs/plugins/repository-s3.asciidoc           |   4 +-
 plugins/repository-s3/build.gradle            |  50 +++-
 .../repositories/s3/S3Service.java            |   7 +-
 .../repositories/s3/AmazonS3Fixture.java      |  16 +-
 .../50_repository_ecs_credentials.yml         | 243 ++++++++++++++++++
 7 files changed, 315 insertions(+), 13 deletions(-)
 create mode 100644 plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index d6477e05b15..b29bb7a8cd3 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -142,6 +142,8 @@ class ClusterConfiguration {
     // there are cases when value depends on task that is not executed yet on configuration stage
     Map<String, Object> systemProperties = new HashMap<>()
 
+    Map<String, Object> environmentVariables = new HashMap<>()
+
     Map<String, Object> settings = new HashMap<>()
 
     Map<String, String> keystoreSettings = new HashMap<>()
@@ -164,6 +166,11 @@ class ClusterConfiguration {
         systemProperties.put(property, value)
     }
 
+    @Input
+    void environment(String variable, Object value) {
+        environmentVariables.put(variable, value)
+    }
+
     @Input
     void setting(String name, Object value) {
         settings.put(name, value)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index 7844ea77fc1..0dd56b86332 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -181,6 +181,7 @@ class NodeInfo {
 
         args.addAll("-E", "node.portsfile=true")
         env = [:]
+        env.putAll(config.environmentVariables)
         for (Map.Entry<String, String> property : System.properties.entrySet()) {
             if (property.key.startsWith('tests.es.')) {
                 args.add("-E")
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 0d73e35f18e..19ead367204 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -13,8 +13,8 @@ include::install_remove.asciidoc[]
 ==== Getting started with AWS
 
 The plugin provides a repository type named `s3` which may be used when creating a repository.
-The repository defaults to using
-http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html[IAM Role]
+The repository defaults to using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html[ECS IAM Role] or
+http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html[EC2 IAM Role]
 credentials for authentication. The only mandatory setting is the bucket name:
 
 [source,js]
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 225d523817e..181891e2056 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -92,11 +92,15 @@ String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
 String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
 String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
 
+String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
+String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
+
 // If all these variables are missing then we are testing against the internal fixture instead, which has the following
 // credentials hard-coded in.
 
 if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath
-        && !s3EC2Bucket && !s3EC2BasePath) {
+        && !s3EC2Bucket && !s3EC2BasePath
+        && !s3ECSBucket && !s3ECSBasePath) {
   s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
   s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
   s3PermanentBucket = 'permanent-bucket-test'
@@ -105,10 +109,14 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
   s3EC2Bucket = 'ec2-bucket-test'
   s3EC2BasePath = 'integration_test'
 
+  s3ECSBucket = 'ecs-bucket-test'
+  s3ECSBasePath = 'integration_test'
+
   useFixture = true
 
 } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath
-        || !s3EC2Bucket || !s3EC2BasePath) {
+        || !s3EC2Bucket || !s3EC2BasePath
+        || !s3ECSBucket || !s3ECSBasePath) {
   throw new IllegalArgumentException("not all options specified to run against external S3 service")
 }
 
@@ -284,7 +292,8 @@ if (useFixture && minioDistribution) {
   // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
   integTestMinioRunner.systemProperty 'tests.rest.blacklist', [
           'repository_s3/30_repository_temporary_credentials/*',
-          'repository_s3/40_repository_ec2_credentials/*'
+          'repository_s3/40_repository_ec2_credentials/*',
+          'repository_s3/50_repository_ecs_credentials/*'
         ].join(",")
 
   project.check.dependsOn(integTestMinio)
@@ -302,7 +311,8 @@ task s3FixtureProperties {
       "s3Fixture.temporary_bucket_name"  : s3TemporaryBucket,
       "s3Fixture.temporary_key"          : s3TemporaryAccessKey,
       "s3Fixture.temporary_session_token": s3TemporarySessionToken,
-      "s3Fixture.ec2_bucket_name"        : s3EC2Bucket
+      "s3Fixture.ec2_bucket_name"        : s3EC2Bucket,
+      "s3Fixture.ecs_bucket_name"        : s3ECSBucket
   ]
 
   doLast {
@@ -327,7 +337,9 @@ Map<String, Object> expansions = [
   'temporary_bucket': s3TemporaryBucket,
   'temporary_base_path': s3TemporaryBasePath,
   'ec2_bucket': s3EC2Bucket,
-  'ec2_base_path': s3EC2BasePath
+  'ec2_base_path': s3EC2BasePath,
+  'ecs_bucket': s3ECSBucket,
+  'ecs_base_path': s3ECSBasePath
 ]
 
 processTestResources {
@@ -364,6 +376,34 @@ integTestCluster {
   }
 }
 
+integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
+
+///
+RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) {
+  description = "Runs tests using the ECS repository."
+}
+
+// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
+project.afterEvaluate {
+  ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration
+  cluster.dependsOn(project.s3Fixture)
+
+  cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}"
+
+  Task integTestECSTask = project.tasks.getByName('integTestECS')
+  integTestECSTask.clusterConfig.plugin(project.path)
+  integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
+          "http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint"
+  integTestECSRunner.systemProperty 'tests.rest.blacklist', [
+          'repository_s3/10_basic/*',
+          'repository_s3/20_repository_permanent_credentials/*',
+          'repository_s3/30_repository_temporary_credentials/*',
+          'repository_s3/40_repository_ec2_credentials/*'
+  ].join(",")
+}
+project.check.dependsOn(integTestECS)
+///
+
 thirdPartyAudit.excludes = [
   // classes are missing
   'javax.servlet.ServletContextEvent', 
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
index 91a7a30024b..b177686bd71 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
@@ -22,7 +22,7 @@ package org.elasticsearch.repositories.s3;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
 import com.amazonaws.http.IdleConnectionReaper;
 import com.amazonaws.internal.StaticCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
@@ -156,10 +156,11 @@ class S3Service extends AbstractComponent implements Closeable {
     }
 
     static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider {
-        private final InstanceProfileCredentialsProvider credentials;
+        private final AWSCredentialsProvider credentials;
 
         private PrivilegedInstanceProfileCredentialsProvider() {
-            this.credentials = new InstanceProfileCredentialsProvider();
+            // InstanceProfileCredentialsProvider as last item of chain
+            this.credentials = new EC2ContainerCredentialsProviderWrapper();
         }
 
         @Override
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java
index ce6c4723149..a411a1c53cf 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java
@@ -88,7 +88,10 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         final Bucket ec2Bucket = new Bucket("s3Fixture.ec2",
             randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10));
 
-        this.handlers = defaultHandlers(buckets, ec2Bucket);
+        final Bucket ecsBucket = new Bucket("s3Fixture.ecs",
+            randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10));
+
+        this.handlers = defaultHandlers(buckets, ec2Bucket, ecsBucket);
     }
 
     private static String nonAuthPath(Request request) {
@@ -174,7 +177,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
     }
 
     /** Builds the default request handlers **/
-    private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> buckets, final Bucket ec2Bucket) {
+    private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> buckets, final Bucket ec2Bucket, final Bucket ecsBucket) {
         final PathTrie<RequestHandler> handlers = new PathTrie<>(RestUtils.REST_DECODER);
 
         // HEAD Object
@@ -400,11 +403,18 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/{profileName}"), (request) -> {
             final String profileName = request.getParam("profileName");
             if (EC2_PROFILE.equals(profileName) == false) {
-                return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown credentials".getBytes(UTF_8));
+                return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown profile".getBytes(UTF_8));
             }
             return credentialResponseFunction.apply(profileName, ec2Bucket.key, ec2Bucket.token);
         });
 
+        // GET
+        //
+        // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
+        handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/ecs_credentials_endpoint"),
+            (request) -> credentialResponseFunction.apply("CPV_ECS", ecsBucket.key, ecsBucket.token));
+
+
         return handlers;
     }
 
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
new file mode 100644
index 00000000000..54929e6e3ad
--- /dev/null
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
@@ -0,0 +1,243 @@
+# Integration tests for repository-s3
+
+---
+setup:
+
+  # Register repository with ecs credentials
+  - do:
+      snapshot.create_repository:
+        repository: repository_ecs
+        body:
+          type: s3
+          settings:
+            bucket: ${ecs_bucket}
+            client: integration_test_ecs
+            base_path: ${ecs_base_path}
+            canned_acl: private
+            storage_class: standard
+
+---
+"Snapshot and Restore with repository-s3 using ecs credentials":
+
+  # Get repository
+  - do:
+      snapshot.get_repository:
+        repository: repository_ecs
+
+  - match: { repository_ecs.settings.bucket : ${ecs_bucket} }
+  - match: { repository_ecs.settings.client : "integration_test_ecs" }
+  - match: { repository_ecs.settings.base_path : ${ecs_base_path} }
+  - match: { repository_ecs.settings.canned_acl : "private" }
+  - match: { repository_ecs.settings.storage_class : "standard" }
+  - is_false: repository_ecs.settings.access_key
+  - is_false: repository_ecs.settings.secret_key
+  - is_false: repository_ecs.settings.session_token
+
+  # Index documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    1
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    2
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    3
+          - snapshot: one
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Create a first snapshot
+  - do:
+      snapshot.create:
+        repository: repository_ecs
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-one }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.include_global_state: true }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.status:
+        repository: repository_ecs
+        snapshot: snapshot-one
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: snapshot-one }
+  - match: { snapshots.0.state : SUCCESS }
+
+  # Index more documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    4
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    5
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    6
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    7
+          - snapshot: two
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Create a second snapshot
+  - do:
+      snapshot.create:
+        repository: repository_ecs
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-two }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.get:
+        repository: repository_ecs
+        snapshot: snapshot-one,snapshot-two
+
+  - is_true: snapshots
+  - match: { snapshots.0.state : SUCCESS }
+  - match: { snapshots.1.state : SUCCESS }
+
+  # Delete the index
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the second snapshot
+  - do:
+      snapshot.restore:
+        repository: repository_ecs
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Delete the index again
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the first snapshot
+  - do:
+      snapshot.restore:
+        repository: repository_ecs
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Remove the snapshots
+  - do:
+      snapshot.delete:
+        repository: repository_ecs
+        snapshot: snapshot-two
+
+  - do:
+      snapshot.delete:
+        repository: repository_ecs
+        snapshot: snapshot-one
+
+---
+"Register a repository with a non existing bucket":
+
+  - do:
+      catch: /repository_exception/
+      snapshot.create_repository:
+        repository: repository_ecs
+        body:
+          type: s3
+          settings:
+            bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+            client: integration_test_temporary
+
+---
+"Register a repository with a non existing client":
+
+  - do:
+      catch: /repository_exception/
+      snapshot.create_repository:
+        repository: repository_ecs
+        body:
+          type: s3
+          settings:
+            bucket: repository_ecs
+            client: unknown
+
+---
+"Get a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.get:
+        repository: repository_ecs
+        snapshot: missing
+
+---
+"Delete a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.delete:
+        repository: repository_ecs
+        snapshot: missing
+
+---
+"Restore a non existing snapshot":
+
+  - do:
+      catch: /snapshot_restore_exception/
+      snapshot.restore:
+        repository: repository_ecs
+        snapshot: missing
+        wait_for_completion: true
+
+---
+teardown:
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: repository_ecs

From 9ae6905657b310ede5d968d71f4f3265eb26db5b Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Thu, 19 Jul 2018 09:17:49 -0700
Subject: [PATCH 103/260] add support for write index resolution when
 creating/updating documents (#31520)

Now write operations like Index, Delete, Update rely on the write-index associated with
an alias to operate against. This means writes will be accepted even when an alias points to multiple indices, so long as one is the write index. Routing values will be used from the AliasMetaData for the alias in the write-index. All read operations are left untouched.
---
 .../action/bulk/TransportBulkAction.java      |   4 +-
 .../action/index/IndexRequest.java            |   2 +-
 .../action/update/TransportUpdateAction.java  |   2 +-
 .../metadata/IndexNameExpressionResolver.java | 101 ++++++++----
 .../cluster/metadata/MetaData.java            |  36 ++++
 .../action/bulk/BulkIntegrationIT.java        |  41 +++++
 .../elasticsearch/aliases/IndexAliasesIT.java |  51 ++++++
 .../IndexNameExpressionResolverTests.java     | 154 ++++++++++++++++++
 .../cluster/metadata/MetaDataTests.java       |  81 +++++++++
 .../org/elasticsearch/get/GetActionIT.java    |  26 ++-
 .../org/elasticsearch/update/UpdateIT.java    |   3 +-
 .../test/security/authz/12_index_alias.yml    |  71 ++++++++
 12 files changed, 534 insertions(+), 38 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index a6ed8de6530..939b0b70249 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -295,7 +295,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                             TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (UpdateRequest) docWriteRequest);
                             break;
                         case DELETE:
-                            docWriteRequest.routing(metaData.resolveIndexRouting(docWriteRequest.routing(), docWriteRequest.index()));
+                            docWriteRequest.routing(metaData.resolveWriteIndexRouting(docWriteRequest.routing(), docWriteRequest.index()));
                             // check if routing is required, if so, throw error if routing wasn't specified
                             if (docWriteRequest.routing() == null && metaData.routingRequired(concreteIndex.getName(), docWriteRequest.type())) {
                                 throw new RoutingMissingException(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id());
@@ -474,7 +474,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
         Index resolveIfAbsent(DocWriteRequest<?> request) {
             Index concreteIndex = indices.get(request.index());
             if (concreteIndex == null) {
-                concreteIndex = indexNameExpressionResolver.concreteSingleIndex(state, request);
+                concreteIndex = indexNameExpressionResolver.concreteWriteIndex(state, request);
                 indices.put(request.index(), concreteIndex);
             }
             return concreteIndex;
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index 51997b32edf..57e8ea66138 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -496,7 +496,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
 
     /* resolve the routing if needed */
     public void resolveRouting(MetaData metaData) {
-        routing(metaData.resolveIndexRouting(routing, index));
+        routing(metaData.resolveWriteIndexRouting(routing, index));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index 299a2ce8123..cc682619cbd 100644
--- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -104,7 +104,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
     }
 
     public static void resolveAndValidateRouting(MetaData metaData, String concreteIndex, UpdateRequest request) {
-        request.routing((metaData.resolveIndexRouting(request.routing(), request.index())));
+        request.routing((metaData.resolveWriteIndexRouting(request.routing(), request.index())));
         // Fail fast on the node that received the request, rather than failing when translating on the index or delete request.
         if (request.routing() == null && metaData.routingRequired(concreteIndex, request.type())) {
             throw new RoutingMissingException(concreteIndex, request.type(), request.id());
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 8fa3c2e0fc1..1f6a9fe027d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -42,7 +42,6 @@ import org.joda.time.format.DateTimeFormatter;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -103,7 +102,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         return concreteIndexNames(context, indexExpressions);
     }
 
-     /**
+    /**
      * Translates the provided index expression into actual concrete indices, properly deduplicated.
      *
      * @param state             the cluster state containing all the data to resolve to expressions to concrete indices
@@ -117,7 +116,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
      * indices options in the context don't allow such a case.
      */
     public Index[] concreteIndices(ClusterState state, IndicesOptions options, String... indexExpressions) {
-        Context context = new Context(state, options);
+        Context context = new Context(state, options, false, false);
         return concreteIndices(context, indexExpressions);
     }
 
@@ -193,30 +192,40 @@ public class IndexNameExpressionResolver extends AbstractComponent {
                 }
             }
 
-            Collection<IndexMetaData> resolvedIndices = aliasOrIndex.getIndices();
-            if (resolvedIndices.size() > 1 && !options.allowAliasesToMultipleIndices()) {
-                String[] indexNames = new String[resolvedIndices.size()];
-                int i = 0;
-                for (IndexMetaData indexMetaData : resolvedIndices) {
-                    indexNames[i++] = indexMetaData.getIndex().getName();
+            if (aliasOrIndex.isAlias() && context.isResolveToWriteIndex()) {
+                AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex;
+                IndexMetaData writeIndex = alias.getWriteIndex();
+                if (writeIndex == null) {
+                    throw new IllegalArgumentException("no write index is defined for alias [" + alias.getAliasName() + "]." +
+                        " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" +
+                        " indices without one being designated as a write index");
                 }
-                throw new IllegalArgumentException("Alias [" + expression + "] has more than one indices associated with it [" +
-                        Arrays.toString(indexNames) + "], can't execute a single index op");
-            }
-
-            for (IndexMetaData index : resolvedIndices) {
-                if (index.getState() == IndexMetaData.State.CLOSE) {
-                    if (failClosed) {
-                        throw new IndexClosedException(index.getIndex());
-                    } else {
-                        if (options.forbidClosedIndices() == false) {
-                            concreteIndices.add(index.getIndex());
-                        }
+                concreteIndices.add(writeIndex.getIndex());
+            } else {
+                if (aliasOrIndex.getIndices().size() > 1 && !options.allowAliasesToMultipleIndices()) {
+                    String[] indexNames = new String[aliasOrIndex.getIndices().size()];
+                    int i = 0;
+                    for (IndexMetaData indexMetaData : aliasOrIndex.getIndices()) {
+                        indexNames[i++] = indexMetaData.getIndex().getName();
+                    }
+                    throw new IllegalArgumentException("Alias [" + expression + "] has more than one indices associated with it [" +
+                        Arrays.toString(indexNames) + "], can't execute a single index op");
+                }
+
+                for (IndexMetaData index : aliasOrIndex.getIndices()) {
+                    if (index.getState() == IndexMetaData.State.CLOSE) {
+                        if (failClosed) {
+                            throw new IndexClosedException(index.getIndex());
+                        } else {
+                            if (options.forbidClosedIndices() == false) {
+                                concreteIndices.add(index.getIndex());
+                            }
+                        }
+                    } else if (index.getState() == IndexMetaData.State.OPEN) {
+                        concreteIndices.add(index.getIndex());
+                    } else {
+                        throw new IllegalStateException("index state [" + index.getState() + "] not supported");
                     }
-                } else if (index.getState() == IndexMetaData.State.OPEN) {
-                    concreteIndices.add(index.getIndex());
-                } else {
-                    throw new IllegalStateException("index state [" + index.getState() + "] not supported");
                 }
             }
         }
@@ -255,6 +264,28 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         return indices[0];
     }
 
+    /**
+     * Utility method that allows to resolve an index expression to its corresponding single write index.
+     *
+     * @param state             the cluster state containing all the data to resolve to expression to a concrete index
+     * @param request           The request that defines how the an alias or an index need to be resolved to a concrete index
+     *                          and the expression that can be resolved to an alias or an index name.
+     * @throws IllegalArgumentException if the index resolution does not lead to an index, or leads to more than one index
+     * @return the write index obtained as a result of the index resolution
+     */
+    public Index concreteWriteIndex(ClusterState state, IndicesRequest request) {
+        if (request.indices() == null || (request.indices() != null && request.indices().length != 1)) {
+            throw new IllegalArgumentException("indices request must specify a single index expression");
+        }
+        Context context = new Context(state, request.indicesOptions(), false, true);
+        Index[] indices = concreteIndices(context, request.indices()[0]);
+        if (indices.length != 1) {
+            throw new IllegalArgumentException("The index expression [" + request.indices()[0] +
+                "] and options provided did not point to a single write-index");
+        }
+        return indices[0];
+    }
+
     /**
      * @return whether the specified alias or index exists. If the alias or index contains datemath then that is resolved too.
      */
@@ -292,7 +323,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
                                  String... expressions) {
         // expand the aliases wildcard
         List<String> resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.emptyList();
-        Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true);
+        Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false);
         for (ExpressionResolver expressionResolver : expressionResolvers) {
             resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
         }
@@ -512,24 +543,26 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         private final IndicesOptions options;
         private final long startTime;
         private final boolean preserveAliases;
+        private final boolean resolveToWriteIndex;
 
         Context(ClusterState state, IndicesOptions options) {
             this(state, options, System.currentTimeMillis());
         }
 
-        Context(ClusterState state, IndicesOptions options, boolean preserveAliases) {
-            this(state, options, System.currentTimeMillis(), preserveAliases);
+        Context(ClusterState state, IndicesOptions options, boolean preserveAliases, boolean resolveToWriteIndex) {
+            this(state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex);
         }
 
         Context(ClusterState state, IndicesOptions options, long startTime) {
-           this(state, options, startTime, false);
+           this(state, options, startTime, false, false);
         }
 
-        Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases) {
+        Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex) {
             this.state = state;
             this.options = options;
             this.startTime = startTime;
             this.preserveAliases = preserveAliases;
+            this.resolveToWriteIndex = resolveToWriteIndex;
         }
 
         public ClusterState getState() {
@@ -552,6 +585,14 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         boolean isPreserveAliases() {
             return preserveAliases;
         }
+
+        /**
+         * This is used to require that aliases resolve to their write-index. It is currently not used in conjunction
+         * with <code>preserveAliases</code>.
+         */
+        boolean isResolveToWriteIndex() {
+            return resolveToWriteIndex;
+        }
     }
 
     private interface ExpressionResolver {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 4ed2adc9a1c..c0243888683 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -471,6 +471,42 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
         return allClosedIndices;
     }
 
+    /**
+     * Returns indexing routing for the given <code>aliasOrIndex</code>. Resolves routing from the alias metadata used
+     * in the write index.
+     */
+    public String resolveWriteIndexRouting(@Nullable String routing, String aliasOrIndex) {
+        if (aliasOrIndex == null) {
+            return routing;
+        }
+
+        AliasOrIndex result = getAliasAndIndexLookup().get(aliasOrIndex);
+        if (result == null || result.isAlias() == false) {
+            return routing;
+        }
+        AliasOrIndex.Alias alias = (AliasOrIndex.Alias) result;
+        IndexMetaData writeIndex = alias.getWriteIndex();
+        if (writeIndex == null) {
+            throw new IllegalArgumentException("alias [" + aliasOrIndex + "] does not have a write index");
+        }
+        AliasMetaData aliasMd = writeIndex.getAliases().get(alias.getAliasName());
+        if (aliasMd.indexRouting() != null) {
+            if (aliasMd.indexRouting().indexOf(',') != -1) {
+                throw new IllegalArgumentException("index/alias [" + aliasOrIndex + "] provided with routing value ["
+                    + aliasMd.getIndexRouting() + "] that resolved to several routing values, rejecting operation");
+            }
+            if (routing != null) {
+                if (!routing.equals(aliasMd.indexRouting())) {
+                    throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has index routing associated with it ["
+                        + aliasMd.indexRouting() + "], and was provided with routing value [" + routing + "], rejecting operation");
+                }
+            }
+            // Alias routing overrides the parent routing (if any).
+            return aliasMd.indexRouting();
+        }
+        return routing;
+    }
+
     /**
      * Returns indexing routing for the given index.
      */
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
index 8fcc76e018a..1fd912e72a4 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
@@ -20,13 +20,20 @@
 
 package org.elasticsearch.action.bulk;
 
+import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Map;
 
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
+import static org.hamcrest.Matchers.equalTo;
 
 public class BulkIntegrationIT extends ESIntegTestCase {
     public void testBulkIndexCreatesMapping() throws Exception {
@@ -40,4 +47,38 @@ public class BulkIntegrationIT extends ESIntegTestCase {
             assertTrue(mappingsResponse.getMappings().get("logstash-2014.03.30").containsKey("logs"));
         });
     }
+
+    /**
+     * This tests that the {@link TransportBulkAction} evaluates alias routing values correctly when dealing with
+     * an alias pointing to multiple indices, while a write index exits.
+     */
+    public void testBulkWithWriteIndexAndRouting() {
+        Map<String, Integer> twoShardsSettings = Collections.singletonMap(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2);
+        client().admin().indices().prepareCreate("index1")
+            .addAlias(new Alias("alias1").indexRouting("0")).setSettings(twoShardsSettings).get();
+        client().admin().indices().prepareCreate("index2")
+            .addAlias(new Alias("alias1").indexRouting("0").writeIndex(randomFrom(false, null)))
+            .setSettings(twoShardsSettings).get();
+        client().admin().indices().prepareCreate("index3")
+            .addAlias(new Alias("alias1").indexRouting("1").writeIndex(true)).setSettings(twoShardsSettings).get();
+
+        IndexRequest indexRequestWithAlias = new IndexRequest("alias1", "type", "id");
+        if (randomBoolean()) {
+            indexRequestWithAlias.routing("1");
+        }
+        indexRequestWithAlias.source(Collections.singletonMap("foo", "baz"));
+        BulkResponse bulkResponse = client().prepareBulk().add(indexRequestWithAlias).get();
+        assertThat(bulkResponse.getItems()[0].getResponse().getIndex(), equalTo("index3"));
+        assertThat(bulkResponse.getItems()[0].getResponse().getShardId().getId(), equalTo(0));
+        assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L));
+        assertThat(bulkResponse.getItems()[0].getResponse().status(), equalTo(RestStatus.CREATED));
+        assertThat(client().prepareGet("index3", "type", "id").setRouting("1").get().getSource().get("foo"), equalTo("baz"));
+
+        bulkResponse = client().prepareBulk().add(client().prepareUpdate("alias1", "type", "id").setDoc("foo", "updated")).get();
+        assertFalse(bulkResponse.hasFailures());
+        assertThat(client().prepareGet("index3", "type", "id").setRouting("1").get().getSource().get("foo"), equalTo("updated"));
+        bulkResponse = client().prepareBulk().add(client().prepareDelete("alias1", "type", "id")).get();
+        assertFalse(bulkResponse.hasFailures());
+        assertFalse(client().prepareGet("index3", "type", "id").setRouting("1").get().isExists());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
index d72b4c5f1ec..e8c152abdc2 100644
--- a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasA
 import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
@@ -57,6 +58,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.client.Requests.createIndexRequest;
+import static org.elasticsearch.client.Requests.deleteRequest;
 import static org.elasticsearch.client.Requests.indexRequest;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_METADATA_BLOCK;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_READ_ONLY_BLOCK;
@@ -85,6 +87,17 @@ public class IndexAliasesIT extends ESIntegTestCase {
 
         ensureGreen();
 
+        logger.info("--> aliasing index [test] with [alias1]");
+        assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1", false));
+
+        logger.info("--> indexing against [alias1], should fail now");
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> client().index(indexRequest("alias1").type("type1").id("1").source(source("2", "test"),
+                XContentType.JSON)).actionGet());
+        assertThat(exception.getMessage(), equalTo("no write index is defined for alias [alias1]." +
+            " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" +
+            " indices without one being designated as a write index"));
+
         logger.info("--> aliasing index [test] with [alias1]");
         assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1"));
 
@@ -98,6 +111,44 @@ public class IndexAliasesIT extends ESIntegTestCase {
 
         ensureGreen();
 
+        logger.info("--> add index [test_x] with [alias1]");
+        assertAcked(admin().indices().prepareAliases().addAlias("test_x", "alias1"));
+
+        logger.info("--> indexing against [alias1], should fail now");
+        exception = expectThrows(IllegalArgumentException.class,
+            () -> client().index(indexRequest("alias1").type("type1").id("1").source(source("2", "test"),
+                XContentType.JSON)).actionGet());
+        assertThat(exception.getMessage(), equalTo("no write index is defined for alias [alias1]." +
+            " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" +
+            " indices without one being designated as a write index"));
+
+        logger.info("--> deleting against [alias1], should fail now");
+        exception = expectThrows(IllegalArgumentException.class,
+            () -> client().delete(deleteRequest("alias1").type("type1").id("1")).actionGet());
+        assertThat(exception.getMessage(), equalTo("no write index is defined for alias [alias1]." +
+            " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" +
+            " indices without one being designated as a write index"));
+
+        logger.info("--> remove aliasing index [test_x] with [alias1]");
+        assertAcked(admin().indices().prepareAliases().removeAlias("test_x", "alias1"));
+
+        logger.info("--> indexing against [alias1], should work now");
+        indexResponse = client().index(indexRequest("alias1").type("type1").id("1")
+            .source(source("1", "test"), XContentType.JSON)).actionGet();
+        assertThat(indexResponse.getIndex(), equalTo("test"));
+
+        logger.info("--> add index [test_x] with [alias1] as write-index");
+        assertAcked(admin().indices().prepareAliases().addAlias("test_x", "alias1", true));
+
+        logger.info("--> indexing against [alias1], should work now");
+        indexResponse = client().index(indexRequest("alias1").type("type1").id("1")
+            .source(source("1", "test"), XContentType.JSON)).actionGet();
+        assertThat(indexResponse.getIndex(), equalTo("test_x"));
+
+        logger.info("--> deleting against [alias1], should fail now");
+        DeleteResponse deleteResponse = client().delete(deleteRequest("alias1").type("type1").id("1")).actionGet();
+        assertThat(deleteResponse.getIndex(), equalTo("test_x"));
+
         logger.info("--> remove [alias1], Aliasing index [test_x] with [alias1]");
         assertAcked(admin().indices().prepareAliases().removeAlias("test", "alias1").addAlias("test_x", "alias1"));
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index 0530bd617af..9ad9603b148 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -20,14 +20,20 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData.State;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.InvalidIndexNameException;
@@ -37,6 +43,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.function.Function;
 
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.hamcrest.Matchers.arrayContaining;
@@ -44,6 +51,7 @@ import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyArray;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -996,6 +1004,152 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         assertArrayEquals(new String[] {"test-alias-0", "test-alias-1", "test-alias-non-filtering"}, strings);
     }
 
+    public void testConcreteWriteIndexSuccessful() {
+        boolean testZeroWriteIndex = randomBoolean();
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("test-0").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias").writeIndex(testZeroWriteIndex ? true : null)));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        String[] strings = indexNameExpressionResolver
+            .indexAliases(state, "test-0", x -> true, true, "test-*");
+        Arrays.sort(strings);
+        assertArrayEquals(new String[] {"test-alias"}, strings);
+        IndicesRequest request =  new IndicesRequest()  {
+
+            @Override
+            public String[] indices() {
+                return new String[] { "test-alias" };
+            }
+
+            @Override
+            public IndicesOptions indicesOptions() {
+                return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+            }
+        };
+        Index writeIndex = indexNameExpressionResolver.concreteWriteIndex(state, request);
+        assertThat(writeIndex.getName(), equalTo("test-0"));
+
+        state = ClusterState.builder(state).metaData(MetaData.builder(state.metaData())
+            .put(indexBuilder("test-1").putAlias(AliasMetaData.builder("test-alias")
+                .writeIndex(testZeroWriteIndex ? randomFrom(false, null) : true)))).build();
+        writeIndex = indexNameExpressionResolver.concreteWriteIndex(state, request);
+        assertThat(writeIndex.getName(), equalTo(testZeroWriteIndex ? "test-0" : "test-1"));
+    }
+
+    public void testConcreteWriteIndexWithInvalidIndicesRequest() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("test-0").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias")));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        Function<String[], IndicesRequest> requestGen = (indices) -> new IndicesRequest()  {
+
+            @Override
+            public String[] indices() {
+                return indices;
+            }
+
+            @Override
+            public IndicesOptions indicesOptions() {
+                return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+            }
+        };
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> indexNameExpressionResolver.concreteWriteIndex(state, requestGen.apply(null)));
+        assertThat(exception.getMessage(), equalTo("indices request must specify a single index expression"));
+        exception = expectThrows(IllegalArgumentException.class,
+            () -> indexNameExpressionResolver.concreteWriteIndex(state, requestGen.apply(new String[] {"too", "many"})));
+        assertThat(exception.getMessage(), equalTo("indices request must specify a single index expression"));
+
+
+    }
+
+    public void testConcreteWriteIndexWithWildcardExpansion() {
+        boolean testZeroWriteIndex = randomBoolean();
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("test-1").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias").writeIndex(testZeroWriteIndex ? true : null)))
+            .put(indexBuilder("test-0").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias").writeIndex(testZeroWriteIndex ? randomFrom(false, null) : true)));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        String[] strings = indexNameExpressionResolver
+            .indexAliases(state, "test-0", x -> true, true, "test-*");
+        Arrays.sort(strings);
+        assertArrayEquals(new String[] {"test-alias"}, strings);
+        IndicesRequest request =  new IndicesRequest()  {
+
+            @Override
+            public String[] indices() {
+                return new String[] { "test-*"};
+            }
+
+            @Override
+            public IndicesOptions indicesOptions() {
+                return IndicesOptions.strictExpandOpenAndForbidClosed();
+            }
+        };
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> indexNameExpressionResolver.concreteWriteIndex(state, request));
+        assertThat(exception.getMessage(),
+            equalTo("The index expression [test-*] and options provided did not point to a single write-index"));
+    }
+
+    public void testConcreteWriteIndexWithNoWriteIndexWithSingleIndex() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("test-0").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias").writeIndex(false)));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        String[] strings = indexNameExpressionResolver
+            .indexAliases(state, "test-0", x -> true, true, "test-*");
+        Arrays.sort(strings);
+        assertArrayEquals(new String[] {"test-alias"}, strings);
+        DocWriteRequest request = randomFrom(new IndexRequest("test-alias"),
+            new UpdateRequest("test-alias", "_type", "_id"), new DeleteRequest("test-alias"));
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> indexNameExpressionResolver.concreteWriteIndex(state, request));
+        assertThat(exception.getMessage(), equalTo("no write index is defined for alias [test-alias]." +
+                " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" +
+                " indices without one being designated as a write index"));
+    }
+
+    public void testConcreteWriteIndexWithNoWriteIndexWithMultipleIndices() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("test-0").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias").writeIndex(randomFrom(false, null))))
+            .put(indexBuilder("test-1").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias").writeIndex(randomFrom(false, null))));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        String[] strings = indexNameExpressionResolver
+            .indexAliases(state, "test-0", x -> true, true, "test-*");
+        Arrays.sort(strings);
+        assertArrayEquals(new String[] {"test-alias"}, strings);
+        DocWriteRequest request = randomFrom(new IndexRequest("test-alias"),
+            new UpdateRequest("test-alias", "_type", "_id"), new DeleteRequest("test-alias"));
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> indexNameExpressionResolver.concreteWriteIndex(state, request));
+        assertThat(exception.getMessage(), equalTo("no write index is defined for alias [test-alias]." +
+            " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" +
+            " indices without one being designated as a write index"));
+    }
+
+    public void testAliasResolutionNotAllowingMultipleIndices() {
+        boolean test0WriteIndex = randomBoolean();
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("test-0").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias").writeIndex(randomFrom(test0WriteIndex, null))))
+            .put(indexBuilder("test-1").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias").writeIndex(randomFrom(!test0WriteIndex, null))));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        String[] strings = indexNameExpressionResolver
+            .indexAliases(state, "test-0", x -> true, true, "test-*");
+        Arrays.sort(strings);
+        assertArrayEquals(new String[] {"test-alias"}, strings);
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed(),
+                "test-alias"));
+        assertThat(exception.getMessage(), endsWith(", can't execute a single index op"));
+    }
+
     public void testDeleteIndexIgnoresAliases() {
         MetaData.Builder mdBuilder = MetaData.builder()
                 .put(indexBuilder("test-index").state(State.OPEN)
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 32dd4324ff8..38e3fcc6ea7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -172,6 +172,87 @@ public class MetaDataTests extends ESTestCase {
         } catch (IllegalArgumentException ex) {
             assertThat(ex.getMessage(), is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation"));
         }
+
+        IndexMetaData.Builder builder2 = IndexMetaData.builder("index2")
+            .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putAlias(AliasMetaData.builder("alias0").build());
+        MetaData metaDataTwoIndices = MetaData.builder(metaData).put(builder2).build();
+
+        // alias with multiple indices
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> metaDataTwoIndices.resolveIndexRouting("1", "alias0"));
+        assertThat(exception.getMessage(), startsWith("Alias [alias0] has more than one index associated with it"));
+    }
+
+    public void testResolveWriteIndexRouting() {
+        AliasMetaData.Builder aliasZeroBuilder = AliasMetaData.builder("alias0");
+        if (randomBoolean()) {
+            aliasZeroBuilder.writeIndex(true);
+        }
+        IndexMetaData.Builder builder = IndexMetaData.builder("index")
+            .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putAlias(aliasZeroBuilder.build())
+            .putAlias(AliasMetaData.builder("alias1").routing("1").build())
+            .putAlias(AliasMetaData.builder("alias2").routing("1,2").build())
+            .putAlias(AliasMetaData.builder("alias3").writeIndex(false).build())
+            .putAlias(AliasMetaData.builder("alias4").routing("1,2").writeIndex(true).build());
+        MetaData metaData = MetaData.builder().put(builder).build();
+
+        // no alias, no index
+        assertEquals(metaData.resolveWriteIndexRouting(null, null), null);
+        assertEquals(metaData.resolveWriteIndexRouting("0", null), "0");
+
+        // index, no alias
+        assertEquals(metaData.resolveWriteIndexRouting(null, "index"), null);
+        assertEquals(metaData.resolveWriteIndexRouting("0", "index"), "0");
+
+        // alias with no index routing
+        assertEquals(metaData.resolveWriteIndexRouting(null, "alias0"), null);
+        assertEquals(metaData.resolveWriteIndexRouting("0", "alias0"), "0");
+
+        // alias with index routing.
+        assertEquals(metaData.resolveWriteIndexRouting(null, "alias1"), "1");
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting("0", "alias1"));
+        assertThat(exception.getMessage(),
+            is("Alias [alias1] has index routing associated with it [1], and was provided with routing value [0], rejecting operation"));
+
+        // alias with invalid index routing.
+        exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting(null, "alias2"));
+            assertThat(exception.getMessage(),
+                is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation"));
+        exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting("1", "alias2"));
+        assertThat(exception.getMessage(),
+            is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation"));
+        exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting(randomFrom("1", null), "alias4"));
+        assertThat(exception.getMessage(),
+            is("index/alias [alias4] provided with routing value [1,2] that resolved to several routing values, rejecting operation"));
+
+        // alias with no write index
+        exception = expectThrows(IllegalArgumentException.class, () -> metaData.resolveWriteIndexRouting("1", "alias3"));
+        assertThat(exception.getMessage(),
+            is("alias [alias3] does not have a write index"));
+
+
+        // aliases with multiple indices
+        AliasMetaData.Builder aliasZeroBuilderTwo = AliasMetaData.builder("alias0");
+        if (randomBoolean()) {
+            aliasZeroBuilder.writeIndex(false);
+        }
+        IndexMetaData.Builder builder2 = IndexMetaData.builder("index2")
+            .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putAlias(aliasZeroBuilderTwo.build())
+            .putAlias(AliasMetaData.builder("alias1").routing("0").writeIndex(true).build())
+            .putAlias(AliasMetaData.builder("alias2").writeIndex(true).build());
+        MetaData metaDataTwoIndices = MetaData.builder(metaData).put(builder2).build();
+
+        // verify that new write index is used
+        assertThat("0", equalTo(metaDataTwoIndices.resolveWriteIndexRouting("0", "alias1")));
     }
 
     public void testUnknownFieldClusterMetaData() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/get/GetActionIT.java b/server/src/test/java/org/elasticsearch/get/GetActionIT.java
index 30f86241cbd..5ed6b957c78 100644
--- a/server/src/test/java/org/elasticsearch/get/GetActionIT.java
+++ b/server/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.get.MultiGetRequestBuilder;
 import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
@@ -39,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 
@@ -51,6 +53,7 @@ import java.util.Set;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
@@ -70,7 +73,7 @@ public class GetActionIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", "field1", "type=keyword,store=true", "field2", "type=keyword,store=true")
                 .setSettings(Settings.builder().put("index.refresh_interval", -1))
-                .addAlias(new Alias("alias")));
+                .addAlias(new Alias("alias").writeIndex(randomFrom(true, false, null))));
         ensureGreen();
 
         GetResponse response = client().prepareGet(indexOrAlias(), "type1", "1").get();
@@ -192,12 +195,31 @@ public class GetActionIT extends ESIntegTestCase {
         assertThat(response.isExists(), equalTo(false));
     }
 
+    public void testGetWithAliasPointingToMultipleIndices() {
+        client().admin().indices().prepareCreate("index1")
+            .addAlias(new Alias("alias1").indexRouting("0")).get();
+        if (randomBoolean()) {
+            client().admin().indices().prepareCreate("index2")
+                .addAlias(new Alias("alias1").indexRouting("0").writeIndex(randomFrom(false, null))).get();
+        } else {
+            client().admin().indices().prepareCreate("index3")
+                .addAlias(new Alias("alias1").indexRouting("1").writeIndex(true)).get();
+        }
+        IndexResponse indexResponse = client().prepareIndex("index1", "type", "id")
+            .setSource(Collections.singletonMap("foo", "bar")).get();
+        assertThat(indexResponse.status().getStatus(), equalTo(RestStatus.CREATED.getStatus()));
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+            client().prepareGet("alias1", "type", "_alias_id").get());
+        assertThat(exception.getMessage(), endsWith("can't execute a single index op"));
+    }
+
     private static String indexOrAlias() {
         return randomBoolean() ? "test" : "alias";
     }
 
     public void testSimpleMultiGet() throws Exception {
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias").writeIndex(randomFrom(true, false, null)))
                 .addMapping("type1", "field", "type=keyword,store=true")
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));
         ensureGreen();
diff --git a/server/src/test/java/org/elasticsearch/update/UpdateIT.java b/server/src/test/java/org/elasticsearch/update/UpdateIT.java
index c86dfcb98f7..e4ea078b8f7 100644
--- a/server/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/server/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -140,8 +140,7 @@ public class UpdateIT extends ESIntegTestCase {
 
     private void createTestIndex() throws Exception {
         logger.info("--> creating index test");
-
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias").writeIndex(randomFrom(true, null))));
     }
 
     public void testUpsert() throws Exception {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/12_index_alias.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/12_index_alias.yml
index 44d91d691e1..1e947c5639d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/12_index_alias.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/12_index_alias.yml
@@ -310,3 +310,74 @@ teardown:
         index: write_index_2
         body: { "query": { "terms": { "_id":  [ "19" ] } } }
   - match: { hits.total: 1 }
+
+---
+"Test bulk indexing into an alias when resolved to write index":
+  - do:
+      indices.update_aliases:
+        body:
+          actions:
+            - add:
+                index: write_index_2
+                alias: can_write_2
+                is_write_index: true
+            - add:
+                index: write_index_2
+                alias: can_read_2
+                is_write_index: true
+            - add:
+                index: write_index_1
+                alias: can_write_3
+                is_write_index: true
+            - add:
+                index: write_index_2
+                alias: can_write_3
+                is_write_index: false
+
+  - do:
+      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "can_read_1", "_type": "doc", "_id": "20"}}'
+            - '{"name": "doc20"}'
+            - '{"index": {"_index": "can_write_1", "_type": "doc", "_id": "21"}}'
+            - '{"name": "doc21"}'
+            - '{"index": {"_index": "can_read_2", "_type": "doc", "_id": "22"}}'
+            - '{"name": "doc22"}'
+            - '{"index": {"_index": "can_write_2", "_type": "doc", "_id": "23"}}'
+            - '{"name": "doc23"}'
+            - '{"index": {"_index": "can_write_3", "_type": "doc", "_id": "24"}}'
+            - '{"name": "doc24"}'
+            - '{"update": {"_index": "can_write_3", "_type": "doc", "_id": "24"}}'
+            - '{"doc": { "name": "doc_24"}}'
+            - '{"delete": {"_index": "can_write_3", "_type": "doc", "_id": "24"}}'
+  - match: { errors: true }
+  - match: { items.0.index.status: 403 }
+  - match: { items.0.index.error.type: "security_exception" }
+  - match: { items.1.index.status: 201 }
+  - match: { items.2.index.status: 403 }
+  - match: { items.2.index.error.type: "security_exception" }
+  - match: { items.3.index.status: 403 }
+  - match: { items.3.index.error.type: "security_exception" }
+  - match: { items.4.index.status: 201 }
+  - match: { items.5.update.status: 200 }
+  - match: { items.6.delete.status: 200 }
+
+  - do: # superuser
+      search:
+        index: write_index_1
+        body: { "query": { "terms": { "_id":  [ "21" ] } } }
+  - match: { hits.total: 1 }
+
+  - do:
+      indices.delete_alias:
+        index: "write_index_2"
+        name: [ "can_write_2", "can_read_2" ]
+        ignore: 404
+
+  - do:
+      indices.delete_alias:
+        index: "write_index_1"
+        name: [ "can_write_3" ]
+        ignore: 404

From 4c68dfe0016673bbdb1885384d1934858180d521 Mon Sep 17 00:00:00 2001
From: Mayya Sharipova <mayya.sharipova@elastic.co>
Date: Thu, 19 Jul 2018 17:41:06 -0400
Subject: [PATCH 104/260] Handle missing values in painless (#32207)

Throw an exception for doc['field'].value
if this document is missing a value for the field.

After deprecation changes have been backported to 6.x,
make this a default behaviour in 7.0

Closes #29286
---
 .../elasticsearch/gradle/BuildPlugin.groovy   |   1 -
 .../painless-getting-started.asciidoc         |  17 +-
 server/build.gradle                           |  10 -
 .../index/fielddata/ScriptDocValues.java      |  44 +---
 .../elasticsearch/script/ScriptModule.java    |  13 +-
 ...criptDocValuesMissingV6BehaviourTests.java | 195 ------------------
 6 files changed, 14 insertions(+), 266 deletions(-)
 delete mode 100644 server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 219d00ba640..c5dd19de3cc 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -750,7 +750,6 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
-            systemProperty 'es.scripting.exception_for_missing_value', 'true'
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {
diff --git a/docs/painless/painless-getting-started.asciidoc b/docs/painless/painless-getting-started.asciidoc
index 887769e49ab..1dec4a33bb5 100644
--- a/docs/painless/painless-getting-started.asciidoc
+++ b/docs/painless/painless-getting-started.asciidoc
@@ -123,21 +123,8 @@ GET hockey/_search
 [float]
 ===== Missing values
 
-If you request the value from a field `field` that isn’t in
-the document, `doc['field'].value` for this document returns:
-
-- `0` if a `field` has a numeric datatype (long, double etc.)
-- `false` is a `field` has a boolean datatype
-- epoch date if a `field` has a date datatype
-- `null` if a `field` has a string datatype
-- `null` if a `field` has a geo datatype
-- `""` if a `field` has a binary datatype
-
-IMPORTANT: Starting in 7.0, `doc['field'].value` throws an exception if
-the field is missing in a document. To enable this behavior now,
-set a {ref}/jvm-options.html[`jvm.option`]
-`-Des.scripting.exception_for_missing_value=true` on a node. If you do not enable
-this behavior, a deprecation warning is logged on start up.
+`doc['field'].value` throws an exception if
+the field is missing in a document.
 
 To check if a document is missing a value, you can call
 `doc['field'].size() == 0`.
diff --git a/server/build.gradle b/server/build.gradle
index 7db073f43a5..c71cc4c7dbd 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -156,16 +156,6 @@ if (isEclipse) {
 compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
-// TODO: remove ScriptDocValuesMissingV6BehaviourTests in 7.0
-additionalTest('testScriptDocValuesMissingV6Behaviour'){
-  include '**/ScriptDocValuesMissingV6BehaviourTests.class'
-  systemProperty 'es.scripting.exception_for_missing_value', 'false'
-}
-test {
-  // these are tested explicitly in separate test tasks
-  exclude '**/*ScriptDocValuesMissingV6BehaviourTests.class'
-}
-
 forbiddenPatterns {
   exclude '**/*.json'
   exclude '**/*.jmx'
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index 6d888bd63e3..fedad6e134b 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.script.ScriptModule;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
@@ -126,11 +125,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
 
         public long getValue() {
             if (count == 0) {
-                if (ScriptModule.EXCEPTION_FOR_MISSING_VALUE) {
-                    throw new IllegalStateException("A document doesn't have a value for a field! " +
-                        "Use doc[<field>].size()==0 to check if a document is missing a field!");
-                }
-                return 0L;
+                throw new IllegalStateException("A document doesn't have a value for a field! " +
+                    "Use doc[<field>].size()==0 to check if a document is missing a field!");
             }
             return values[0];
         }
@@ -172,11 +168,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
          */
         public ReadableDateTime getValue() {
             if (count == 0) {
-                if (ScriptModule.EXCEPTION_FOR_MISSING_VALUE) {
-                    throw new IllegalStateException("A document doesn't have a value for a field! " +
-                        "Use doc[<field>].size()==0 to check if a document is missing a field!");
-                }
-                return EPOCH;
+                throw new IllegalStateException("A document doesn't have a value for a field! " +
+                    "Use doc[<field>].size()==0 to check if a document is missing a field!");
             }
             return get(0);
         }
@@ -277,11 +270,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
 
         public double getValue() {
             if (count == 0) {
-                if (ScriptModule.EXCEPTION_FOR_MISSING_VALUE) {
-                    throw new IllegalStateException("A document doesn't have a value for a field! " +
-                        "Use doc[<field>].size()==0 to check if a document is missing a field!");
-                }
-                return 0d;
+               throw new IllegalStateException("A document doesn't have a value for a field! " +
+                   "Use doc[<field>].size()==0 to check if a document is missing a field!");
             }
             return values[0];
         }
@@ -337,11 +327,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
 
         public GeoPoint getValue() {
             if (count == 0) {
-                if (ScriptModule.EXCEPTION_FOR_MISSING_VALUE) {
-                    throw new IllegalStateException("A document doesn't have a value for a field! " +
+                throw new IllegalStateException("A document doesn't have a value for a field! " +
                         "Use doc[<field>].size()==0 to check if a document is missing a field!");
-                }
-                return null;
             }
             return values[0];
         }
@@ -454,11 +441,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
 
         public boolean getValue() {
             if (count == 0) {
-                if (ScriptModule.EXCEPTION_FOR_MISSING_VALUE) {
-                    throw new IllegalStateException("A document doesn't have a value for a field! " +
-                        "Use doc[<field>].size()==0 to check if a document is missing a field!");
-                }
-                return false;
+                throw new IllegalStateException("A document doesn't have a value for a field! " +
+                    "Use doc[<field>].size()==0 to check if a document is missing a field!");
             }
             return values[0];
         }
@@ -544,11 +528,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
 
         public String getValue() {
             if (count == 0) {
-                if (ScriptModule.EXCEPTION_FOR_MISSING_VALUE) {
-                    throw new IllegalStateException("A document doesn't have a value for a field! " +
+                throw new IllegalStateException("A document doesn't have a value for a field! " +
                         "Use doc[<field>].size()==0 to check if a document is missing a field!");
-                }
-                return null;
             }
             return get(0);
         }
@@ -572,11 +553,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
 
         public BytesRef getValue() {
             if (count == 0) {
-                if (ScriptModule.EXCEPTION_FOR_MISSING_VALUE) {
-                    throw new IllegalStateException("A document doesn't have a value for a field! " +
+                throw new IllegalStateException("A document doesn't have a value for a field! " +
                         "Use doc[<field>].size()==0 to check if a document is missing a field!");
-                }
-                return new BytesRef();
             }
             return get(0);
         }
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java
index bf4bd9c57ce..a3da1dafe48 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -31,9 +31,7 @@ import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.search.aggregations.pipeline.movfn.MovingFunctionScript;
-import org.elasticsearch.common.Booleans;
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
+
 
 /**
  * Manages building {@link ScriptService}.
@@ -64,11 +62,6 @@ public class ScriptModule {
         ).collect(Collectors.toMap(c -> c.name, Function.identity()));
     }
 
-    public static final boolean EXCEPTION_FOR_MISSING_VALUE =
-        Booleans.parseBoolean(System.getProperty("es.scripting.exception_for_missing_value", "false"));
-
-    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(ScriptModule.class));
-
     private final ScriptService scriptService;
 
     public ScriptModule(Settings settings, List<ScriptPlugin> scriptPlugins) {
@@ -92,10 +85,6 @@ public class ScriptModule {
                 }
             }
         }
-        if (EXCEPTION_FOR_MISSING_VALUE == false)
-            DEPRECATION_LOGGER.deprecated("Script: returning default values for missing document values is deprecated. " +
-                    "Set system property '-Des.scripting.exception_for_missing_value=true' " +
-                    "to make behaviour compatible with future major versions.");
         scriptService = new ScriptService(settings, Collections.unmodifiableMap(engines), Collections.unmodifiableMap(contexts));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java
deleted file mode 100644
index 1dc836874d8..00000000000
--- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata;
-
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.fielddata.ScriptDocValues.Longs;
-import org.elasticsearch.index.fielddata.ScriptDocValues.Dates;
-import org.elasticsearch.index.fielddata.ScriptDocValues.Booleans;
-import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.MockScriptEngine;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptEngine;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.test.ESTestCase;
-
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.ReadableDateTime;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-
-import static java.util.Collections.singletonList;
-
-public class ScriptDocValuesMissingV6BehaviourTests extends ESTestCase {
-
-    public void testScriptMissingValuesWarning(){
-        new ScriptModule(Settings.EMPTY, singletonList(new ScriptPlugin() {
-            @Override
-            public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
-                return new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1"));
-            }
-        }));
-        assertWarnings("Script: returning default values for missing document values is deprecated. " +
-            "Set system property '-Des.scripting.exception_for_missing_value=true' " +
-            "to make behaviour compatible with future major versions.");
-    }
-
-    public void testZeroForMissingValueLong() throws IOException {
-        long[][] values = new long[between(3, 10)][];
-        for (int d = 0; d < values.length; d++) {
-            values[d] = new long[0];
-        }
-        Longs longs = wrap(values);
-        for (int round = 0; round < 10; round++) {
-            int d = between(0, values.length - 1);
-            longs.setNextDocId(d);
-            assertEquals(0, longs.getValue());
-        }
-    }
-
-    public void testEpochForMissingValueDate() throws IOException {
-        final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC);
-        long[][] values = new long[between(3, 10)][];
-        for (int d = 0; d < values.length; d++) {
-            values[d] = new long[0];
-        }
-        Dates dates = wrapDates(values);
-        for (int round = 0; round < 10; round++) {
-            int d = between(0, values.length - 1);
-            dates.setNextDocId(d);
-            assertEquals(EPOCH, dates.getValue());
-        }
-    }
-
-    public void testFalseForMissingValueBoolean() throws IOException {
-        long[][] values = new long[between(3, 10)][];
-        for (int d = 0; d < values.length; d++) {
-            values[d] = new long[0];
-        }
-        Booleans bools = wrapBooleans(values);
-        for (int round = 0; round < 10; round++) {
-            int d = between(0, values.length - 1);
-            bools.setNextDocId(d);
-            assertEquals(false, bools.getValue());
-        }
-    }
-
-    public void testNullForMissingValueGeo() throws IOException{
-        final MultiGeoPointValues values = wrap(new GeoPoint[0]);
-        final ScriptDocValues.GeoPoints script = new ScriptDocValues.GeoPoints(values);
-        script.setNextDocId(0);
-        assertEquals(null, script.getValue());
-    }
-
-
-    private Longs wrap(long[][] values) {
-        return new Longs(new AbstractSortedNumericDocValues() {
-            long[] current;
-            int i;
-            @Override
-            public boolean advanceExact(int doc) {
-                i = 0;
-                current = values[doc];
-                return current.length > 0;
-            }
-            @Override
-            public int docValueCount() {
-                return current.length;
-            }
-            @Override
-            public long nextValue() {
-                return current[i++];
-            }
-        });
-    }
-
-    private Booleans wrapBooleans(long[][] values) {
-        return new Booleans(new AbstractSortedNumericDocValues() {
-            long[] current;
-            int i;
-            @Override
-            public boolean advanceExact(int doc) {
-                i = 0;
-                current = values[doc];
-                return current.length > 0;
-            }
-            @Override
-            public int docValueCount() {
-                return current.length;
-            }
-            @Override
-            public long nextValue() {
-                return current[i++];
-            }
-        });
-    }
-
-    private Dates wrapDates(long[][] values) {
-        return new Dates(new AbstractSortedNumericDocValues() {
-            long[] current;
-            int i;
-            @Override
-            public boolean advanceExact(int doc) {
-                current = values[doc];
-                i = 0;
-                return current.length > 0;
-            }
-            @Override
-            public int docValueCount() {
-                return current.length;
-            }
-            @Override
-            public long nextValue() {
-                return current[i++];
-            }
-        });
-    }
-
-
-   private static MultiGeoPointValues wrap(final GeoPoint... points) {
-        return new MultiGeoPointValues() {
-            int docID = -1;
-            int i;
-            @Override
-            public GeoPoint nextValue() {
-                if (docID != 0) {
-                    fail();
-                }
-                return points[i++];
-            }
-            @Override
-            public boolean advanceExact(int docId) {
-                docID = docId;
-                return points.length > 0;
-            }
-            @Override
-            public int docValueCount() {
-                if (docID != 0) {
-                    return 0;
-                }
-                return points.length;
-            }
-        };
-    }
-
-}

From c7a41c501ac0c8ef6a6cd96c2abc7c4fbd7fb76e Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 19 Jul 2018 16:35:03 -0700
Subject: [PATCH 105/260] Painless: Simplify Naming in Lookup Package (#32177)

This removes some extraneous naming syntax and makes clear the meaning of certain
naming conventions without ambiguities (stricter) within the lookup package. Purely
mechanical change. Note this does not cover a large portion of the
PainlessLookupBuilder and PainlessLookup yet as there are several more follow up PRs for these incoming.
---
 .../painless/AnalyzerCaster.java              |   4 +-
 .../java/org/elasticsearch/painless/Def.java  |   6 +-
 .../elasticsearch/painless/FunctionRef.java   |   2 +-
 .../org/elasticsearch/painless/Locals.java    |   2 +-
 .../painless/ScriptClassInfo.java             |   2 +-
 .../painless/lookup/PainlessLookup.java       |   2 +-
 .../lookup/PainlessLookupBuilder.java         | 248 ++++------
 .../lookup/PainlessLookupUtility.java         | 466 ++++++++++--------
 .../painless/lookup/PainlessMethod.java       |  14 +-
 .../painless/node/AExpression.java            |   2 +-
 .../elasticsearch/painless/node/EBinary.java  |  44 +-
 .../painless/node/ECapturingFunctionRef.java  |   8 +-
 .../elasticsearch/painless/node/ECast.java    |   2 +-
 .../elasticsearch/painless/node/EComp.java    |  32 +-
 .../painless/node/EFunctionRef.java           |   4 +-
 .../painless/node/EInstanceof.java            |   6 +-
 .../elasticsearch/painless/node/ELambda.java  |  10 +-
 .../elasticsearch/painless/node/ENull.java    |   2 +-
 .../elasticsearch/painless/node/EUnary.java   |   6 +-
 .../elasticsearch/painless/node/PBrace.java   |   2 +-
 .../painless/node/PCallInvoke.java            |   2 +-
 .../elasticsearch/painless/node/PField.java   |   7 +-
 .../painless/node/PSubField.java              |   2 +-
 .../elasticsearch/painless/node/SEach.java    |   2 +-
 .../painless/node/SFunction.java              |   4 +-
 .../painless/node/SSubEachArray.java          |   2 +-
 .../painless/node/SSubEachIterable.java       |   4 +-
 .../painless/PainlessDocGenerator.java        |   4 +-
 28 files changed, 464 insertions(+), 427 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 6cfc7ff6ebf..fe53a3c1100 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -466,8 +466,8 @@ public final class AnalyzerCaster {
             return PainlessCast.standard(actual, expected, explicit);
         } else {
             throw location.createError(new ClassCastException("Cannot cast from " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(actual) + "] to " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] to " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]."));
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index f3388fc4bb2..dad8da06e76 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -302,7 +302,7 @@ public final class Def {
                                                               nestedType,
                                                               0,
                                                               DefBootstrap.REFERENCE,
-                                                              PainlessLookupUtility.anyTypeToPainlessTypeName(interfaceType));
+                                                              PainlessLookupUtility.typeToCanonicalTypeName(interfaceType));
                      filter = nested.dynamicInvoker();
                  } else {
                      throw new AssertionError();
@@ -334,7 +334,7 @@ public final class Def {
          int arity = interfaceMethod.arguments.size();
          PainlessMethod implMethod = lookupMethodInternal(painlessLookup, receiverClass, name, arity);
         return lookupReferenceInternal(painlessLookup, methodHandlesLookup, interfaceType,
-                PainlessLookupUtility.anyTypeToPainlessTypeName(implMethod.target), implMethod.name, receiverClass);
+                PainlessLookupUtility.typeToCanonicalTypeName(implMethod.target), implMethod.name, receiverClass);
      }
 
      /** Returns a method handle to an implementation of clazz, given method reference signature. */
@@ -347,7 +347,7 @@ public final class Def {
              PainlessMethod interfaceMethod = painlessLookup.getPainlessStructFromJavaClass(clazz).functionalMethod;
              if (interfaceMethod == null) {
                  throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                         "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(clazz) + "], not a functional interface");
+                         "to [" + PainlessLookupUtility.typeToCanonicalTypeName(clazz) + "], not a functional interface");
              }
              int arity = interfaceMethod.arguments.size() + captures.length;
              final MethodHandle handle;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index d64e833912f..aa72724b930 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -168,7 +168,7 @@ public class FunctionRef {
         PainlessMethod method = painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod;
         if (method == null) {
             throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                    "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], not a functional interface");
+                    "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface");
         }
 
         // lookup requested method
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index 6c1010a3450..804f6aa2b68 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -292,7 +292,7 @@ public final class Locals {
         @Override
         public String toString() {
             StringBuilder b = new StringBuilder();
-            b.append("Variable[type=").append(PainlessLookupUtility.anyTypeToPainlessTypeName(clazz));
+            b.append("Variable[type=").append(PainlessLookupUtility.typeToCanonicalTypeName(clazz));
             b.append(",name=").append(name);
             b.append(",slot=").append(slot);
             if (readonly) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
index ff2061a9a4b..6d4b4552696 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java
@@ -183,7 +183,7 @@ public class ScriptClassInfo {
 
     private static Class<?> definitionTypeForClass(PainlessLookup painlessLookup, Class<?> type,
                                                    Function<Class<?>, String> unknownErrorMessageSource) {
-        type = PainlessLookupUtility.javaObjectTypeToPainlessDefType(type);
+        type = PainlessLookupUtility.javaTypeToType(type);
         Class<?> componentType = type;
 
         while (componentType.isArray()) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
index 6111d12317b..752c0c205dd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
@@ -54,6 +54,6 @@ public final class PainlessLookup {
     }
 
     public Class<?> getJavaClassFromPainlessType(String painlessType) {
-        return PainlessLookupUtility.painlessTypeNameToPainlessType(painlessType, painlessTypesToJavaClasses);
+        return PainlessLookupUtility.canonicalTypeNameToType(painlessType, painlessTypesToJavaClasses);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index ecf15c7ad2c..06773d3ffdd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -30,7 +30,6 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -39,22 +38,21 @@ import java.util.Objects;
 import java.util.Stack;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_PAINLESS_CLASS_NAME;
-import static org.elasticsearch.painless.lookup.PainlessLookupUtility.anyTypeNameToPainlessTypeName;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_TYPE_NAME;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
 
 public class PainlessLookupBuilder {
 
     private static class PainlessMethodCacheKey {
 
-        private final Class<?> javaClass;
+        private final Class<?> targetType;
         private final String methodName;
-        private final List<Class<?>> painlessTypeParameters;
+        private final List<Class<?>> typeParameters;
 
-        private PainlessMethodCacheKey(Class<?> javaClass, String methodName, List<Class<?>> painlessTypeParameters) {
-            this.javaClass = javaClass;
+        private PainlessMethodCacheKey(Class<?> targetType, String methodName, List<Class<?>> typeParameters) {
+            this.targetType = targetType;
             this.methodName = methodName;
-            this.painlessTypeParameters = Collections.unmodifiableList(painlessTypeParameters);
+            this.typeParameters = Collections.unmodifiableList(typeParameters);
         }
 
         @Override
@@ -69,27 +67,27 @@ public class PainlessLookupBuilder {
 
             PainlessMethodCacheKey that = (PainlessMethodCacheKey)object;
 
-            return Objects.equals(javaClass, that.javaClass)   &&
+            return Objects.equals(targetType, that.targetType) &&
                    Objects.equals(methodName, that.methodName) &&
-                   Objects.equals(painlessTypeParameters, that.painlessTypeParameters);
+                   Objects.equals(typeParameters, that.typeParameters);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(javaClass, methodName, painlessTypeParameters);
+            return Objects.hash(targetType, methodName, typeParameters);
         }
     }
 
     private static class PainlessFieldCacheKey {
 
-        private final Class<?> javaClass;
+        private final Class<?> targetType;
         private final String fieldName;
-        private final Class<?> painlessType;
+        private final Class<?> typeParameter;
 
-        private PainlessFieldCacheKey(Class<?> javaClass, String fieldName, Class<?> painlessType) {
-            this.javaClass = javaClass;
+        private PainlessFieldCacheKey(Class<?> targetType, String fieldName, Class<?> typeParameter) {
+            this.targetType = targetType;
             this.fieldName = fieldName;
-            this.painlessType = painlessType;
+            this.typeParameter = typeParameter;
         }
 
         @Override
@@ -104,14 +102,14 @@ public class PainlessLookupBuilder {
 
             PainlessFieldCacheKey that = (PainlessFieldCacheKey) object;
 
-            return Objects.equals(javaClass, that.javaClass) &&
-                   Objects.equals(fieldName, that.fieldName) &&
-                   Objects.equals(painlessType, that.painlessType);
+            return Objects.equals(targetType, that.targetType) &&
+                   Objects.equals(fieldName, that.fieldName)   &&
+                   Objects.equals(typeParameter, that.typeParameter);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(javaClass, fieldName, painlessType);
+            return Objects.hash(targetType, fieldName, typeParameter);
         }
     }
 
@@ -122,157 +120,115 @@ public class PainlessLookupBuilder {
     private static final Pattern METHOD_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][_a-zA-Z0-9]*$");
     private static final Pattern FIELD_NAME_PATTERN  = Pattern.compile("^[_a-zA-Z][_a-zA-Z0-9]*$");
 
-    private static String anyTypesArrayToCanonicalString(Class<?>[] anyTypesArray, boolean toPainlessTypes) {
-        return anyTypesListToCanonicalString(Arrays.asList(anyTypesArray), toPainlessTypes);
-    }
-
-    private static String anyTypesListToCanonicalString(List<Class<?>> anyTypesList, boolean toPainlessTypes) {
-        StringBuilder anyTypesCanonicalStringBuilder = new StringBuilder("[");
-
-        int anyTypesSize = anyTypesList.size();
-        int anyTypesIndex = 0;
-
-        for (Class<?> anyType : anyTypesList) {
-            String anyTypeCanonicalName = anyType.getCanonicalName();
-
-            if (toPainlessTypes) {
-                anyTypeCanonicalName = anyTypeNameToPainlessTypeName(anyTypeCanonicalName);
-            }
-
-            anyTypesCanonicalStringBuilder.append(anyTypeCanonicalName);
-
-            if (++anyTypesIndex < anyTypesSize) {
-                anyTypesCanonicalStringBuilder.append(",");
-            }
-        }
-
-        anyTypesCanonicalStringBuilder.append("]");
-
-        return anyTypesCanonicalStringBuilder.toString();
-    }
-
     private final List<Whitelist> whitelists;
 
-    private final Map<String, Class<?>> painlessClassNamesToJavaClasses;
-    private final Map<Class<?>, PainlessClassBuilder> javaClassesToPainlessClassBuilders;
+    private final Map<String, Class<?>> canonicalClassNamesToClasses;
+    private final Map<Class<?>, PainlessClassBuilder> classesToPainlessClasses;
 
     public PainlessLookupBuilder(List<Whitelist> whitelists) {
         this.whitelists = whitelists;
 
-        painlessClassNamesToJavaClasses = new HashMap<>();
-        javaClassesToPainlessClassBuilders = new HashMap<>();
+        canonicalClassNamesToClasses = new HashMap<>();
+        classesToPainlessClasses = new HashMap<>();
 
-        painlessClassNamesToJavaClasses.put(DEF_PAINLESS_CLASS_NAME, def.class);
-        javaClassesToPainlessClassBuilders.put(def.class,
-                new PainlessClassBuilder(DEF_PAINLESS_CLASS_NAME, Object.class, Type.getType(Object.class)));
+        canonicalClassNamesToClasses.put(DEF_TYPE_NAME, def.class);
+        classesToPainlessClasses.put(def.class,
+                new PainlessClassBuilder(DEF_TYPE_NAME, Object.class, Type.getType(Object.class)));
     }
 
-    private Class<?> painlessTypeNameToPainlessType(String painlessTypeName) {
-        return PainlessLookupUtility.painlessTypeNameToPainlessType(painlessTypeName, painlessClassNamesToJavaClasses);
+    private Class<?> canonicalTypeNameToType(String canonicalTypeName) {
+        return PainlessLookupUtility.canonicalTypeNameToType(canonicalTypeName, canonicalClassNamesToClasses);
     }
 
-    private void validatePainlessType(Class<?> painlessType) {
-        PainlessLookupUtility.validatePainlessType(painlessType, javaClassesToPainlessClassBuilders.keySet());
+    private void validateType(Class<?> type) {
+        PainlessLookupUtility.validateType(type, classesToPainlessClasses.keySet());
     }
 
-    public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importPainlessClassName) {
+    public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importClassName) {
         Objects.requireNonNull(classLoader);
         Objects.requireNonNull(javaClassName);
 
-        String painlessClassName = anyTypeNameToPainlessTypeName(javaClassName);
+        Class<?> clazz;
 
-        if (CLASS_NAME_PATTERN.matcher(painlessClassName).matches() == false) {
-            throw new IllegalArgumentException("invalid painless class name [" + painlessClassName + "]");
-        }
-
-        String importedPainlessClassName = anyTypeNameToPainlessTypeName(javaClassName.substring(javaClassName.lastIndexOf('.') + 1));
-
-        Class<?> javaClass;
-
-        if ("void".equals(javaClassName)) javaClass = void.class;
-        else if ("boolean".equals(javaClassName)) javaClass = boolean.class;
-        else if ("byte".equals(javaClassName)) javaClass = byte.class;
-        else if ("short".equals(javaClassName)) javaClass = short.class;
-        else if ("char".equals(javaClassName)) javaClass = char.class;
-        else if ("int".equals(javaClassName)) javaClass = int.class;
-        else if ("long".equals(javaClassName)) javaClass = long.class;
-        else if ("float".equals(javaClassName)) javaClass = float.class;
-        else if ("double".equals(javaClassName)) javaClass = double.class;
+        if      ("void".equals(javaClassName))    clazz = void.class;
+        else if ("boolean".equals(javaClassName)) clazz = boolean.class;
+        else if ("byte".equals(javaClassName))    clazz = byte.class;
+        else if ("short".equals(javaClassName))   clazz = short.class;
+        else if ("char".equals(javaClassName))    clazz = char.class;
+        else if ("int".equals(javaClassName))     clazz = int.class;
+        else if ("long".equals(javaClassName))    clazz = long.class;
+        else if ("float".equals(javaClassName))   clazz = float.class;
+        else if ("double".equals(javaClassName))  clazz = double.class;
         else {
             try {
-                javaClass = Class.forName(javaClassName, true, classLoader);
-
-                if (javaClass == def.class) {
-                    throw new IllegalArgumentException("cannot add reserved painless class [" + DEF_PAINLESS_CLASS_NAME + "]");
-                }
-
-                if (javaClass.isArray()) {
-                    throw new IllegalArgumentException("cannot add an array type java class [" + javaClassName + "] as a painless class");
-                }
+                clazz = Class.forName(javaClassName, true, classLoader);
             } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("java class [" + javaClassName + "] not found", cnfe);
+                throw new IllegalArgumentException("class [" + javaClassName + "] not found", cnfe);
             }
         }
 
-        addPainlessClass(painlessClassName, importedPainlessClassName, javaClass, importPainlessClassName);
+        addPainlessClass(clazz, importClassName);
     }
 
-    public void addPainlessClass(Class<?> javaClass, boolean importPainlessClassName) {
-        Objects.requireNonNull(javaClass);
+    public void addPainlessClass(Class<?> clazz, boolean importClassName) {
+        Objects.requireNonNull(clazz);
 
-        if (javaClass == def.class) {
-            throw new IllegalArgumentException("cannot specify reserved painless class [" + DEF_PAINLESS_CLASS_NAME + "]");
+        if (clazz == def.class) {
+            throw new IllegalArgumentException("cannot add reserved class [" + DEF_TYPE_NAME + "]");
         }
 
-        String javaClassName = javaClass.getCanonicalName();
-        String painlessClassName = anyTypeNameToPainlessTypeName(javaClassName);
-        String importedPainlessClassName = anyTypeNameToPainlessTypeName(javaClassName.substring(javaClassName.lastIndexOf('.') + 1));
+        String canonicalClassName = clazz.getCanonicalName();
 
-        addPainlessClass(painlessClassName, importedPainlessClassName, javaClass, importPainlessClassName);
-    }
+        if (clazz.isArray()) {
+            throw new IllegalArgumentException("cannot add array type [" + canonicalClassName + "] as a class");
+        }
 
-    private void addPainlessClass(
-        String painlessClassName, String importedPainlessClassName, Class<?> javaClass, boolean importPainlessClassName) {
-        PainlessClassBuilder existingPainlessClassBuilder = javaClassesToPainlessClassBuilders.get(javaClass);
+        if (CLASS_NAME_PATTERN.matcher(canonicalClassName).matches() == false) {
+            throw new IllegalArgumentException("invalid class name [" + canonicalClassName + "]");
+        }
+
+        PainlessClassBuilder existingPainlessClassBuilder = classesToPainlessClasses.get(clazz);
 
         if (existingPainlessClassBuilder == null) {
-            PainlessClassBuilder painlessClassBuilder = new PainlessClassBuilder(painlessClassName, javaClass, Type.getType(javaClass));
-            painlessClassNamesToJavaClasses.put(painlessClassName, javaClass);
-            javaClassesToPainlessClassBuilders.put(javaClass, painlessClassBuilder);
-        } else if (existingPainlessClassBuilder.clazz.equals(javaClass) == false) {
-            throw new IllegalArgumentException("painless class [" + painlessClassName + "] illegally represents multiple java classes " +
-                    "[" + javaClass.getCanonicalName() + "] and [" + existingPainlessClassBuilder.clazz.getCanonicalName() + "]");
+            PainlessClassBuilder painlessClassBuilder = new PainlessClassBuilder(canonicalClassName, clazz, Type.getType(clazz));
+
+            canonicalClassNamesToClasses.put(canonicalClassName, clazz);
+            classesToPainlessClasses.put(clazz, painlessClassBuilder);
+        } else if (existingPainlessClassBuilder.clazz.equals(clazz) == false) {
+            throw new IllegalArgumentException("class [" + canonicalClassName + "] " +
+                    "cannot represent multiple java classes with the same name from different class loaders");
         }
 
-        if (painlessClassName.equals(importedPainlessClassName)) {
-            if (importPainlessClassName == true) {
-                throw new IllegalArgumentException(
-                        "must use only_fqn parameter on painless class [" + painlessClassName + "] with no package");
+        String javaClassName = clazz.getName();
+        String importedCanonicalClassName = javaClassName.substring(javaClassName.lastIndexOf('.') + 1).replace('$', '.');
+
+        if (canonicalClassName.equals(importedCanonicalClassName)) {
+            if (importClassName == true) {
+                throw new IllegalArgumentException("must use only_fqn parameter on class [" + canonicalClassName + "] with no package");
             }
         } else {
-            Class<?> importedJavaClass = painlessClassNamesToJavaClasses.get(importedPainlessClassName);
+            Class<?> importedPainlessType = canonicalClassNamesToClasses.get(importedCanonicalClassName);
 
-            if (importedJavaClass == null) {
-                if (importPainlessClassName) {
+            if (importedPainlessType == null) {
+                if (importClassName) {
                     if (existingPainlessClassBuilder != null) {
                         throw new IllegalArgumentException(
-                                "inconsistent only_fqn parameters found for painless class [" + painlessClassName + "]");
+                                "inconsistent only_fqn parameters found for painless type [" + canonicalClassName + "]");
                     }
 
-                    painlessClassNamesToJavaClasses.put(importedPainlessClassName, javaClass);
+                    canonicalClassNamesToClasses.put(importedCanonicalClassName, clazz);
                 }
-            } else if (importedJavaClass.equals(javaClass) == false) {
-                throw new IllegalArgumentException("painless class [" + importedPainlessClassName + "] illegally represents multiple " +
-                        "java classes [" + javaClass.getCanonicalName() + "] and [" + importedJavaClass.getCanonicalName() + "]");
-            } else if (importPainlessClassName == false) {
-                throw new IllegalArgumentException(
-                        "inconsistent only_fqn parameters found for painless class [" + painlessClassName + "]");
+            } else if (importedPainlessType.equals(clazz) == false) {
+                throw new IllegalArgumentException("painless type [" + importedCanonicalClassName + "] illegally represents multiple " +
+                        "java types [" + clazz.getCanonicalName() + "] and [" + importedPainlessType.getCanonicalName() + "]");
+            } else if (importClassName == false) {
+                throw new IllegalArgumentException("inconsistent only_fqn parameters found for painless type [" + canonicalClassName + "]");
             }
         }
     }
 
     private void addConstructor(String ownerStructName, WhitelistConstructor whitelistConstructor) {
-        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for constructor with " +
@@ -286,10 +242,10 @@ public class PainlessLookupBuilder {
             String painlessParameterTypeName = whitelistConstructor.painlessParameterTypeNames.get(parameterCount);
 
             try {
-                Class<?> painlessParameterClass = painlessTypeNameToPainlessType(painlessParameterTypeName);
+                Class<?> painlessParameterClass = canonicalTypeNameToType(painlessParameterTypeName);
 
                 painlessParametersTypes.add(painlessParameterClass);
-                javaClassParameters[parameterCount] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(painlessParameterClass);
+                javaClassParameters[parameterCount] = PainlessLookupUtility.typeToJavaType(painlessParameterClass);
             } catch (IllegalArgumentException iae) {
                 throw new IllegalArgumentException("struct not defined for constructor parameter [" + painlessParameterTypeName + "] " +
                     "with owner struct [" + ownerStructName + "] and constructor parameters " +
@@ -333,7 +289,7 @@ public class PainlessLookupBuilder {
     }
 
     private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, WhitelistMethod whitelistMethod) {
-        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
@@ -372,11 +328,11 @@ public class PainlessLookupBuilder {
             String painlessParameterTypeName = whitelistMethod.painlessParameterTypeNames.get(parameterCount);
 
             try {
-                Class<?> painlessParameterClass = painlessTypeNameToPainlessType(painlessParameterTypeName);
+                Class<?> painlessParameterClass = canonicalTypeNameToType(painlessParameterTypeName);
 
                 painlessParametersTypes.add(painlessParameterClass);
                 javaClassParameters[parameterCount + augmentedOffset] =
-                        PainlessLookupUtility.painlessDefTypeToJavaObjectType(painlessParameterClass);
+                        PainlessLookupUtility.typeToJavaType(painlessParameterClass);
             } catch (IllegalArgumentException iae) {
                 throw new IllegalArgumentException("struct not defined for method parameter [" + painlessParameterTypeName + "] " +
                     "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
@@ -398,14 +354,14 @@ public class PainlessLookupBuilder {
         Class<?> painlessReturnClass;
 
         try {
-            painlessReturnClass = painlessTypeNameToPainlessType(whitelistMethod.painlessReturnTypeName);
+            painlessReturnClass = canonicalTypeNameToType(whitelistMethod.painlessReturnTypeName);
         } catch (IllegalArgumentException iae) {
             throw new IllegalArgumentException("struct not defined for return type [" + whitelistMethod.painlessReturnTypeName + "] " +
                 "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
                 "and parameters " + whitelistMethod.painlessParameterTypeNames, iae);
         }
 
-        if (javaMethod.getReturnType() != PainlessLookupUtility.painlessDefTypeToJavaObjectType(painlessReturnClass)) {
+        if (javaMethod.getReturnType() != PainlessLookupUtility.typeToJavaType(painlessReturnClass)) {
             throw new IllegalArgumentException("specified return type class [" + painlessReturnClass + "] " +
                 "does not match the return type class [" + javaMethod.getReturnType() + "] for the " +
                 "method with name [" + whitelistMethod.javaMethodName + "] " +
@@ -471,7 +427,7 @@ public class PainlessLookupBuilder {
     }
 
     private void addField(String ownerStructName, WhitelistField whitelistField) {
-        PainlessClassBuilder ownerStruct = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(ownerStructName));
+        PainlessClassBuilder ownerStruct = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(ownerStructName));
 
         if (ownerStruct == null) {
             throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
@@ -495,7 +451,7 @@ public class PainlessLookupBuilder {
         Class<?> painlessFieldClass;
 
         try {
-            painlessFieldClass = painlessTypeNameToPainlessType(whitelistField.painlessFieldTypeName);
+            painlessFieldClass = canonicalTypeNameToType(whitelistField.painlessFieldTypeName);
         } catch (IllegalArgumentException iae) {
             throw new IllegalArgumentException("struct not defined for return type [" + whitelistField.painlessFieldTypeName + "] " +
                 "with owner struct [" + ownerStructName + "] and field with name [" + whitelistField.javaFieldName + "]", iae);
@@ -552,7 +508,7 @@ public class PainlessLookupBuilder {
     }
 
     private void copyStruct(String struct, List<String> children) {
-        final PainlessClassBuilder owner = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(struct));
+        final PainlessClassBuilder owner = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(struct));
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
@@ -560,7 +516,7 @@ public class PainlessLookupBuilder {
 
         for (int count = 0; count < children.size(); ++count) {
             final PainlessClassBuilder child =
-                    javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(children.get(count)));
+                    classesToPainlessClasses.get(canonicalClassNamesToClasses.get(children.get(count)));
 
             if (child == null) {
                 throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
@@ -734,7 +690,7 @@ public class PainlessLookupBuilder {
                 for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
                     String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
                     PainlessClassBuilder painlessStruct =
-                        javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(painlessTypeName));
+                        classesToPainlessClasses.get(canonicalClassNamesToClasses.get(painlessTypeName));
 
                     if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
                         throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
@@ -745,8 +701,8 @@ public class PainlessLookupBuilder {
                     addPainlessClass(
                             whitelist.javaClassLoader, whitelistStruct.javaClassName, whitelistStruct.onlyFQNJavaClassName == false);
 
-                    painlessStruct = javaClassesToPainlessClassBuilders.get(painlessClassNamesToJavaClasses.get(painlessTypeName));
-                    javaClassesToPainlessClassBuilders.put(painlessStruct.clazz, painlessStruct);
+                    painlessStruct = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(painlessTypeName));
+                    classesToPainlessClasses.put(painlessStruct.clazz, painlessStruct);
                 }
             }
 
@@ -779,8 +735,8 @@ public class PainlessLookupBuilder {
 
         // goes through each Painless struct and determines the inheritance list,
         // and then adds all inherited types to the Painless struct's whitelist
-        for (Class<?> javaClass : javaClassesToPainlessClassBuilders.keySet()) {
-            PainlessClassBuilder painlessStruct = javaClassesToPainlessClassBuilders.get(javaClass);
+        for (Class<?> javaClass : classesToPainlessClasses.keySet()) {
+            PainlessClassBuilder painlessStruct = classesToPainlessClasses.get(javaClass);
 
             List<String> painlessSuperStructs = new ArrayList<>();
             Class<?> javaSuperClass = painlessStruct.clazz.getSuperclass();
@@ -791,7 +747,7 @@ public class PainlessLookupBuilder {
             // adds super classes to the inheritance list
             if (javaSuperClass != null && javaSuperClass.isInterface() == false) {
                 while (javaSuperClass != null) {
-                    PainlessClassBuilder painlessSuperStruct = javaClassesToPainlessClassBuilders.get(javaSuperClass);
+                    PainlessClassBuilder painlessSuperStruct = classesToPainlessClasses.get(javaSuperClass);
 
                     if (painlessSuperStruct != null) {
                         painlessSuperStructs.add(painlessSuperStruct.name);
@@ -807,7 +763,7 @@ public class PainlessLookupBuilder {
                 Class<?> javaInterfaceLookup = javaInteraceLookups.pop();
 
                 for (Class<?> javaSuperInterface : javaInterfaceLookup.getInterfaces()) {
-                    PainlessClassBuilder painlessInterfaceStruct = javaClassesToPainlessClassBuilders.get(javaSuperInterface);
+                    PainlessClassBuilder painlessInterfaceStruct = classesToPainlessClasses.get(javaSuperInterface);
 
                     if (painlessInterfaceStruct != null) {
                         String painlessInterfaceStructName = painlessInterfaceStruct.name;
@@ -828,7 +784,7 @@ public class PainlessLookupBuilder {
 
             // copies methods and fields from Object into interface types
             if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) {
-                PainlessClassBuilder painlessObjectStruct = javaClassesToPainlessClassBuilders.get(Object.class);
+                PainlessClassBuilder painlessObjectStruct = classesToPainlessClasses.get(Object.class);
 
                 if (painlessObjectStruct != null) {
                     copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name));
@@ -837,18 +793,18 @@ public class PainlessLookupBuilder {
         }
 
         // precompute runtime classes
-        for (PainlessClassBuilder painlessStruct : javaClassesToPainlessClassBuilders.values()) {
+        for (PainlessClassBuilder painlessStruct : classesToPainlessClasses.values()) {
             addRuntimeClass(painlessStruct);
         }
 
         Map<Class<?>, PainlessClass> javaClassesToPainlessClasses = new HashMap<>();
 
         // copy all structs to make them unmodifiable for outside users:
-        for (Map.Entry<Class<?>,PainlessClassBuilder> entry : javaClassesToPainlessClassBuilders.entrySet()) {
+        for (Map.Entry<Class<?>,PainlessClassBuilder> entry : classesToPainlessClasses.entrySet()) {
             entry.getValue().functionalMethod = computeFunctionalInterfaceMethod(entry.getValue());
             javaClassesToPainlessClasses.put(entry.getKey(), entry.getValue().build());
         }
 
-        return new PainlessLookup(painlessClassNamesToJavaClasses, javaClassesToPainlessClasses);
+        return new PainlessLookup(canonicalClassNamesToClasses, javaClassesToPainlessClasses);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
index 0f7c8fb915c..1f698b7c673 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
@@ -19,35 +19,177 @@
 
 package org.elasticsearch.painless.lookup;
 
-import org.objectweb.asm.Type;
-
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
- * This class contains methods shared by {@link PainlessLookupBuilder}, {@link PainlessLookup}, and other classes within
+ * PainlessLookupUtility contains methods shared by {@link PainlessLookupBuilder}, {@link PainlessLookup}, and other classes within
  * Painless for conversion between type names and types along with some other various utility methods.
  *
  * The following terminology is used for variable names throughout the lookup package:
  *
- * - javaClass           (Class)         - a java class including def and excluding array type java classes
- * - javaClassName       (String)        - the fully qualified java class name for a javaClass
- * - painlessClassName   (String)        - the fully qualified painless name or imported painless name for a painlessClass
- * - anyClassName        (String)        - either a javaClassName or a painlessClassName
- * - javaType            (Class)         - a java class excluding def and array type java classes
- * - painlessType        (Class)         - a java class including def and array type java classes
- * - javaTypeName        (String)        - the fully qualified java Type name for a javaType
- * - painlessTypeName    (String)        - the fully qualified painless name or imported painless name for a painlessType
- * - anyTypeName         (String)        - either a javaTypeName or a painlessTypeName
- * - painlessClass       (PainlessClass) - a painless class object
+ * A class is a set of methods and fields under a specific class name. A type is either a class or an array under a specific type name.
+ * Note the distinction between class versus type is class means that no array classes will be be represented whereas type allows array
+ * classes to be represented.  The set of available classes will always be a subset of the available types.
  *
- * Under ambiguous circumstances most variable names are prefixed with asm, java, or painless.
- * If the variable name is the same for asm, java, and painless, no prefix is used.
+ * Under ambiguous circumstances most variable names are prefixed with asm, java, or painless. If the variable value is the same for asm,
+ * java, and painless, no prefix is used.
+ *
+ * <ul>
+ *     <li> - javaClassName     (String)         - the fully qualified java class name where '$' tokens represent inner classes excluding
+ *                                                 def and array types </li>
+ *
+ *     <li> - javaClass          (Class)         - a java class excluding def and array types </li>
+ *
+ *     <li> - javaType           (Class)         - a java class excluding def and including array types </li>
+ *
+ *     <li> - importedClassName (String)         - the imported painless class name where the java canonical class name is used without
+ *                                                 the package qualifier
+ *
+ *     <li> - canonicalClassName (String)        - the fully qualified painless class name equivalent to the fully
+ *                                                 qualified java canonical class name or imported painless class name for a class
+ *                                                 including def and excluding array types where '.' tokens represent inner classes <li>
+ *
+ *     <li> - canonicalTypeName (String)         - the fully qualified painless type name equivalent to the fully
+ *                                                 qualified java canonical type name or imported painless type name for a type
+ *                                                 including def where '.' tokens represent inner classes and each set of '[]' tokens
+ *                                                 at the end of the type name represent a single dimension for an array type <li>
+ *
+ *     <li> - class/clazz       (Class)          - a painless class represented by a java class including def and excluding array
+ *                                                 types </li>
+ *
+ *     <li> - type              (Class)          - a painless type represented by a java class including def and array types </li>
+ *
+ *     <li> - painlessClass     (PainlessClass)  - a painless class object </li>
+ *
+ *     <li> - painlessMethod    (PainlessMethod) - a painless method object </li>
+ *
+ *     <li> - painlessField     (PainlessField)  - a painless field object </li>
+ * </ul>
  */
 public final class PainlessLookupUtility {
 
-    public static Class<?> javaObjectTypeToPainlessDefType(Class<?> javaType) {
+    /**
+     * Converts a canonical type name to a type based on the terminology specified as part of the documentation for
+     * {@link PainlessLookupUtility}. Since canonical class names are a subset of canonical type names, this method will
+     * safely convert a canonical class name to a class as well.
+     */
+    public static Class<?> canonicalTypeNameToType(String canonicalTypeName, Map<String, Class<?>> canonicalClassNamesToClasses) {
+        Objects.requireNonNull(canonicalTypeName);
+        Objects.requireNonNull(canonicalClassNamesToClasses);
+
+        Class<?> type = canonicalClassNamesToClasses.get(canonicalTypeName);
+
+        if (type != null) {
+            return type;
+        }
+
+        int arrayDimensions = 0;
+        int arrayIndex = canonicalTypeName.indexOf('[');
+
+        if (arrayIndex != -1) {
+            int typeNameLength = canonicalTypeName.length();
+
+            while (arrayIndex < typeNameLength) {
+                if (canonicalTypeName.charAt(arrayIndex) == '[' &&
+                    ++arrayIndex < typeNameLength  &&
+                    canonicalTypeName.charAt(arrayIndex++) == ']') {
+                    ++arrayDimensions;
+                } else {
+                    throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
+                }
+            }
+
+            canonicalTypeName = canonicalTypeName.substring(0, canonicalTypeName.indexOf('['));
+            type = canonicalClassNamesToClasses.get(canonicalTypeName);
+
+            char arrayBraces[] = new char[arrayDimensions];
+            Arrays.fill(arrayBraces, '[');
+            String javaTypeName = new String(arrayBraces);
+
+            if (type == boolean.class) {
+                javaTypeName += "Z";
+            } else if (type == byte.class) {
+                javaTypeName += "B";
+            } else if (type == short.class) {
+                javaTypeName += "S";
+            } else if (type == char.class) {
+                javaTypeName += "C";
+            } else if (type == int.class) {
+                javaTypeName += "I";
+            } else if (type == long.class) {
+                javaTypeName += "J";
+            } else if (type == float.class) {
+                javaTypeName += "F";
+            } else if (type == double.class) {
+                javaTypeName += "D";
+            } else {
+                javaTypeName += "L" + type.getName() + ";";
+            }
+
+            try {
+                return Class.forName(javaTypeName);
+            } catch (ClassNotFoundException cnfe) {
+                throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found", cnfe);
+            }
+        }
+
+        throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
+    }
+
+    /**
+     * Converts a type to a canonical type name based on the terminology specified as part of the documentation for
+     * {@link PainlessLookupUtility}. Since classes are a subset of types, this method will safely convert a class
+     * to a canonical class name as well.
+     */
+    public static String typeToCanonicalTypeName(Class<?> type) {
+        Objects.requireNonNull(type);
+
+        String canonicalTypeName = type.getCanonicalName();
+
+        if (canonicalTypeName.startsWith(def.class.getName())) {
+            canonicalTypeName = canonicalTypeName.replace(def.class.getName(), DEF_TYPE_NAME);
+        }
+
+        return canonicalTypeName;
+    }
+
+    /**
+     * Converts a list of types to a list of canonical type names as a string based on the terminology specified as part of the
+     * documentation for {@link PainlessLookupUtility}. Since classes are a subset of types, this method will safely convert a list
+     * of classes or a mixed list of classes and types to a list of canonical type names as a string as well.
+     */
+    public static String typesToCanonicalTypeNames(List<Class<?>> types) {
+        StringBuilder typesStringBuilder = new StringBuilder("[");
+
+        int anyTypesSize = types.size();
+        int anyTypesIndex = 0;
+
+        for (Class<?> painlessType : types) {
+            String canonicalTypeName = typeToCanonicalTypeName(painlessType);
+
+            typesStringBuilder.append(canonicalTypeName);
+
+            if (++anyTypesIndex < anyTypesSize) {
+                typesStringBuilder.append(",");
+            }
+        }
+
+        typesStringBuilder.append("]");
+
+        return typesStringBuilder.toString();
+    }
+    /**
+     * Converts a java type to a type based on the terminology specified as part of {@link PainlessLookupUtility} where if a type is an
+     * object class or object array, the returned type will be the equivalent def class or def array. Otherwise, this behaves as an
+     * identity function.
+     */
+    public static Class<?> javaTypeToType(Class<?> javaType) {
+        Objects.requireNonNull(javaType);
+
         if (javaType.isArray()) {
             Class<?> javaTypeComponent = javaType.getComponentType();
             int arrayDimensions = 1;
@@ -58,14 +200,11 @@ public final class PainlessLookupUtility {
             }
 
             if (javaTypeComponent == Object.class) {
-                char[] asmDescriptorBraces = new char[arrayDimensions];
-                Arrays.fill(asmDescriptorBraces, '[');
-
-                String asmDescriptor = new String(asmDescriptorBraces) + Type.getType(def.class).getDescriptor();
-                Type asmType = Type.getType(asmDescriptor);
+                char[] arrayBraces = new char[arrayDimensions];
+                Arrays.fill(arrayBraces, '[');
 
                 try {
-                    return Class.forName(asmType.getInternalName().replace('/', '.'));
+                    return Class.forName(new String(arrayBraces) + "L" + def.class.getName() + ";");
                 } catch (ClassNotFoundException cnfe) {
                     throw new IllegalStateException("internal error", cnfe);
                 }
@@ -77,206 +216,147 @@ public final class PainlessLookupUtility {
         return javaType;
     }
 
-    public static Class<?> painlessDefTypeToJavaObjectType(Class<?> painlessType) {
-        if (painlessType.isArray()) {
-            Class<?> painlessTypeComponent = painlessType.getComponentType();
+    /**
+     * Converts a type to a java type based on the terminology specified as part of {@link PainlessLookupUtility} where if a type is a
+     * def class or def array, the returned type will be the equivalent object class or object array. Otherwise, this behaves as an
+     * identity function.
+     */
+    public static Class<?> typeToJavaType(Class<?> type) {
+        Objects.requireNonNull(type);
+
+        if (type.isArray()) {
+            Class<?> typeComponent = type.getComponentType();
             int arrayDimensions = 1;
 
-            while (painlessTypeComponent.isArray()) {
-                painlessTypeComponent = painlessTypeComponent.getComponentType();
+            while (typeComponent.isArray()) {
+                typeComponent = typeComponent.getComponentType();
                 ++arrayDimensions;
             }
 
-            if (painlessTypeComponent == def.class) {
-                char[] asmDescriptorBraces = new char[arrayDimensions];
-                Arrays.fill(asmDescriptorBraces, '[');
-
-                String asmDescriptor = new String(asmDescriptorBraces) + Type.getType(Object.class).getDescriptor();
-                Type asmType = Type.getType(asmDescriptor);
+            if (typeComponent == def.class) {
+                char[] arrayBraces = new char[arrayDimensions];
+                Arrays.fill(arrayBraces, '[');
 
                 try {
-                    return Class.forName(asmType.getInternalName().replace('/', '.'));
-                } catch (ClassNotFoundException exception) {
-                    throw new IllegalStateException("internal error", exception);
+                    return Class.forName(new String(arrayBraces) + "L" + Object.class.getName() + ";");
+                } catch (ClassNotFoundException cnfe) {
+                    throw new IllegalStateException("internal error", cnfe);
                 }
             }
-        } else if (painlessType == def.class) {
+        } else if (type == def.class) {
             return Object.class;
         }
 
-        return painlessType;
+        return type;
     }
 
-    public static String anyTypeNameToPainlessTypeName(String anyTypeName) {
-        return anyTypeName.replace(def.class.getName(), DEF_PAINLESS_CLASS_NAME).replace('$', '.');
-    }
+    /**
+     * Ensures a type exists based on the terminology specified as part of {@link PainlessLookupUtility}.  Throws an
+     * {@link IllegalArgumentException} if the type does not exist.
+     */
+    public static void validateType(Class<?> type, Collection<Class<?>> classes) {
+        String canonicalTypeName = typeToCanonicalTypeName(type);
 
-    public static String anyTypeToPainlessTypeName(Class<?> anyType) {
-        if (anyType.isLocalClass() || anyType.isAnonymousClass()) {
-            return null;
-        } else if (anyType.isArray()) {
-            Class<?> anyTypeComponent = anyType.getComponentType();
-            int arrayDimensions = 1;
-
-            while (anyTypeComponent.isArray()) {
-                anyTypeComponent = anyTypeComponent.getComponentType();
-                ++arrayDimensions;
-            }
-
-            if (anyTypeComponent == def.class) {
-                StringBuilder painlessDefTypeNameArrayBuilder = new StringBuilder(DEF_PAINLESS_CLASS_NAME);
-
-                for (int dimension = 0; dimension < arrayDimensions; dimension++) {
-                    painlessDefTypeNameArrayBuilder.append("[]");
-                }
-
-                return painlessDefTypeNameArrayBuilder.toString();
-            }
-        } else if (anyType == def.class) {
-            return DEF_PAINLESS_CLASS_NAME;
+        while (type.getComponentType() != null) {
+            type = type.getComponentType();
         }
 
-        return anyType.getCanonicalName().replace('$', '.');
-    }
-
-    public static Class<?> painlessTypeNameToPainlessType(String painlessTypeName, Map<String, Class<?>> painlessClassNamesToJavaClasses) {
-        Class<?> javaClass = painlessClassNamesToJavaClasses.get(painlessTypeName);
-
-        if (javaClass != null) {
-            return javaClass;
-        }
-
-        int arrayDimensions = 0;
-        int arrayIndex = painlessTypeName.indexOf('[');
-
-        if (arrayIndex != -1) {
-            int painlessTypeNameLength = painlessTypeName.length();
-
-            while (arrayIndex < painlessTypeNameLength) {
-                if (painlessTypeName.charAt(arrayIndex) == '[' &&
-                    ++arrayIndex < painlessTypeNameLength  &&
-                    painlessTypeName.charAt(arrayIndex++) == ']') {
-                    ++arrayDimensions;
-                } else {
-                    throw new IllegalArgumentException("painless type [" + painlessTypeName + "] not found");
-                }
-            }
-
-            painlessTypeName = painlessTypeName.substring(0, painlessTypeName.indexOf('['));
-            javaClass = painlessClassNamesToJavaClasses.get(painlessTypeName);
-
-            char javaDescriptorBraces[] = new char[arrayDimensions];
-            Arrays.fill(javaDescriptorBraces, '[');
-            String javaDescriptor = new String(javaDescriptorBraces);
-
-            if (javaClass == boolean.class) {
-                javaDescriptor += "Z";
-            } else if (javaClass == byte.class) {
-                javaDescriptor += "B";
-            } else if (javaClass == short.class) {
-                javaDescriptor += "S";
-            } else if (javaClass == char.class) {
-                javaDescriptor += "C";
-            } else if (javaClass == int.class) {
-                javaDescriptor += "I";
-            } else if (javaClass == long.class) {
-                javaDescriptor += "J";
-            } else if (javaClass == float.class) {
-                javaDescriptor += "F";
-            } else if (javaClass == double.class) {
-                javaDescriptor += "D";
-            } else {
-                javaDescriptor += "L" + javaClass.getName() + ";";
-            }
-
-            try {
-                return Class.forName(javaDescriptor);
-            } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("painless type [" + painlessTypeName + "] not found", cnfe);
-            }
-        }
-
-        throw new IllegalArgumentException("painless type [" + painlessTypeName + "] not found");
-    }
-
-    public static void validatePainlessType(Class<?> painlessType, Collection<Class<?>> javaClasses) {
-        String painlessTypeName = anyTypeNameToPainlessTypeName(painlessType.getName());
-
-        while (painlessType.getComponentType() != null) {
-            painlessType = painlessType.getComponentType();
-        }
-
-        if (javaClasses.contains(painlessType) == false) {
-            throw new IllegalArgumentException("painless type [" + painlessTypeName + "] not found");
+        if (classes.contains(type) == false) {
+            throw new IllegalArgumentException("type [" + canonicalTypeName + "] not found");
         }
     }
 
+    /**
+     * Converts a type  to its boxed type equivalent if one exists based on the terminology specified as part of
+     * {@link PainlessLookupUtility}. Otherwise, this behaves as an identity function.
+     */
+    public static Class<?> typeToBoxedType(Class<?> type) {
+        if (type == boolean.class) {
+            return Boolean.class;
+        } else if (type == byte.class) {
+            return Byte.class;
+        } else if (type == short.class) {
+            return Short.class;
+        } else if (type == char.class) {
+            return Character.class;
+        } else if (type == int.class) {
+            return Integer.class;
+        } else if (type == long.class) {
+            return Long.class;
+        } else if (type == float.class) {
+            return Float.class;
+        } else if (type == double.class) {
+            return Double.class;
+        }
+
+        return type;
+    }
+
+    /**
+     * Converts a type to its unboxed type equivalent if one exists based on the terminology specified as part of
+     * {@link PainlessLookupUtility}. Otherwise, this behaves as an identity function.
+     */
+    public static Class<?> typeToUnboxedType(Class<?> type) {
+        if (type == Boolean.class) {
+            return boolean.class;
+        } else if (type == Byte.class) {
+            return byte.class;
+        } else if (type == Short.class) {
+            return short.class;
+        } else if (type == Character.class) {
+            return char.class;
+        } else if (type == Integer.class) {
+            return int.class;
+        } else if (type == Long.class) {
+            return long.class;
+        } else if (type == Float.class) {
+            return float.class;
+        } else if (type == Double.class) {
+            return double.class;
+        }
+
+        return type;
+    }
+
+    /**
+     * Checks if a type based on the terminology specified as part of {@link PainlessLookupUtility} is available as a constant type
+     * where {@code true} is returned if the type is a constant type and {@code false} otherwise.
+     */
+    public static boolean isConstantType(Class<?> type) {
+        return type == boolean.class ||
+               type == byte.class    ||
+               type == short.class   ||
+               type == char.class    ||
+               type == int.class     ||
+               type == long.class    ||
+               type == float.class   ||
+               type == double.class  ||
+               type == String.class;
+    }
+
+    /**
+     * Constructs a painless method key used to lookup painless methods from a painless class.
+     */
     public static String buildPainlessMethodKey(String methodName, int methodArity) {
         return methodName + "/" + methodArity;
     }
 
+    /**
+     * Constructs a painless field key used to lookup painless fields from a painless class.
+     */
     public static String buildPainlessFieldKey(String fieldName) {
         return fieldName;
     }
 
-    public static Class<?> getBoxedAnyType(Class<?> anyType) {
-        if (anyType == boolean.class) {
-            return Boolean.class;
-        } else if (anyType == byte.class) {
-            return Byte.class;
-        } else if (anyType == short.class) {
-            return Short.class;
-        } else if (anyType == char.class) {
-            return Character.class;
-        } else if (anyType == int.class) {
-            return Integer.class;
-        } else if (anyType == long.class) {
-            return Long.class;
-        } else if (anyType == float.class) {
-            return Float.class;
-        } else if (anyType == double.class) {
-            return Double.class;
-        }
+    /**
+     * The def type name as specified in the source for a script.
+     */
+    public static final String DEF_TYPE_NAME = "def";
 
-        return anyType;
-    }
-
-    public static Class<?> getUnboxedAnyType(Class<?> anyType) {
-        if (anyType == Boolean.class) {
-            return boolean.class;
-        } else if (anyType == Byte.class) {
-            return byte.class;
-        } else if (anyType == Short.class) {
-            return short.class;
-        } else if (anyType == Character.class) {
-            return char.class;
-        } else if (anyType == Integer.class) {
-            return int.class;
-        } else if (anyType == Long.class) {
-            return long.class;
-        } else if (anyType == Float.class) {
-            return float.class;
-        } else if (anyType == Double.class) {
-            return double.class;
-        }
-
-        return anyType;
-    }
-
-    public static boolean isAnyTypeConstant(Class<?> anyType) {
-        return anyType == boolean.class ||
-               anyType == byte.class    ||
-               anyType == short.class   ||
-               anyType == char.class    ||
-               anyType == int.class     ||
-               anyType == long.class    ||
-               anyType == float.class   ||
-               anyType == double.class  ||
-               anyType == String.class;
-    }
-
-    public static final String DEF_PAINLESS_CLASS_NAME = def.class.getSimpleName();
-    public static final String CONSTRUCTOR_ANY_NAME = "<init>";
+    /**
+     * The method name for all constructors.
+     */
+    public static final String CONSTRUCTOR_NAME = "<init>";
 
     private PainlessLookupUtility() {
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
index 2b0d44e7176..3321de94a26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
@@ -70,21 +70,21 @@ public class PainlessMethod {
             params = new Class<?>[1 + arguments.size()];
             params[0] = augmentation;
             for (int i = 0; i < arguments.size(); i++) {
-                params[i + 1] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
+                params[i + 1] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
             }
-            returnValue = PainlessLookupUtility.painlessDefTypeToJavaObjectType(rtn);
+            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
         } else if (Modifier.isStatic(modifiers)) {
             // static method: straightforward copy
             params = new Class<?>[arguments.size()];
             for (int i = 0; i < arguments.size(); i++) {
-                params[i] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
+                params[i] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
             }
-            returnValue = PainlessLookupUtility.painlessDefTypeToJavaObjectType(rtn);
+            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
         } else if ("<init>".equals(name)) {
             // constructor: returns the owner class
             params = new Class<?>[arguments.size()];
             for (int i = 0; i < arguments.size(); i++) {
-                params[i] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
+                params[i] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
             }
             returnValue = target;
         } else {
@@ -92,9 +92,9 @@ public class PainlessMethod {
             params = new Class<?>[1 + arguments.size()];
             params[0] = target;
             for (int i = 0; i < arguments.size(); i++) {
-                params[i + 1] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(arguments.get(i));
+                params[i + 1] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
             }
-            returnValue = PainlessLookupUtility.painlessDefTypeToJavaObjectType(rtn);
+            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
         }
         return MethodType.methodType(returnValue, params);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index dd813f73c3d..ddf289564b1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -157,7 +157,7 @@ public abstract class AExpression extends ANode {
 
                 return ecast;
             } else {
-                if (PainlessLookupUtility.isAnyTypeConstant(expected)) {
+                if (PainlessLookupUtility.isConstantType(expected)) {
                     // For the case where a cast is required, a constant is set,
                     // and the constant can be immediately cast to the expected type.
                     // An EConstant replaces this node with the constant cast appropriately
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 65776ca76f1..00abe788bf4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -106,8 +106,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply multiply [*] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -149,8 +149,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply divide [/] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -197,8 +197,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply remainder [%] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -245,8 +245,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply add [+] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -304,8 +304,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply subtract [-] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -363,8 +363,8 @@ public final class EBinary extends AExpression {
 
         if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply left shift [<<] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote = lhspromote;
@@ -411,8 +411,8 @@ public final class EBinary extends AExpression {
 
         if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply right shift [>>] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote = lhspromote;
@@ -462,8 +462,8 @@ public final class EBinary extends AExpression {
 
         if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         if (lhspromote == def.class || rhspromote == def.class) {
@@ -506,8 +506,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply and [&] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -546,8 +546,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply xor [^] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote;
@@ -587,8 +587,8 @@ public final class EBinary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply or [|] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         actual = promote;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index e8ad9d85ed6..7b35bc1b48e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -69,7 +69,7 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda
                 defPointer = "D" + variable + "." + call + ",1";
             } else {
                 // typed implementation
-                defPointer = "S" + PainlessLookupUtility.anyTypeToPainlessTypeName(captured.clazz) + "." + call + ",1";
+                defPointer = "S" + PainlessLookupUtility.typeToCanonicalTypeName(captured.clazz) + "." + call + ",1";
             }
             actual = String.class;
         } else {
@@ -77,8 +77,8 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda
             // static case
             if (captured.clazz != def.class) {
                 try {
-                    ref = new FunctionRef(
-                            locals.getPainlessLookup(), expected, PainlessLookupUtility.anyTypeToPainlessTypeName(captured.clazz), call, 1);
+                    ref = new FunctionRef(locals.getPainlessLookup(), expected,
+                            PainlessLookupUtility.typeToCanonicalTypeName(captured.clazz), call, 1);
 
                     // check casts between the interface method and the delegate method are legal
                     for (int i = 0; i < ref.interfaceMethod.arguments.size(); ++i) {
@@ -110,7 +110,7 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda
             // typed interface, dynamic implementation
             writer.visitVarInsn(MethodWriter.getType(captured.clazz).getOpcode(Opcodes.ILOAD), captured.getSlot());
             Type methodType = Type.getMethodType(MethodWriter.getType(expected), MethodWriter.getType(captured.clazz));
-            writer.invokeDefCall(call, methodType, DefBootstrap.REFERENCE, PainlessLookupUtility.anyTypeToPainlessTypeName(expected));
+            writer.invokeDefCall(call, methodType, DefBootstrap.REFERENCE, PainlessLookupUtility.typeToCanonicalTypeName(expected));
         } else {
             // typed interface, typed implementation
             writer.visitVarInsn(MethodWriter.getType(captured.clazz).getOpcode(Opcodes.ILOAD), captured.getSlot());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index b0451b685b5..b07613714b8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -63,6 +63,6 @@ final class ECast extends AExpression {
 
     @Override
     public String toString() {
-        return singleLineToString(PainlessLookupUtility.anyTypeToPainlessTypeName(cast.to), child);
+        return singleLineToString(PainlessLookupUtility.typeToCanonicalTypeName(cast.to), child);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index edf18f501bc..4d8a71ae3eb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -93,8 +93,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply equals [==] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -143,8 +143,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply reference equals [===] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         left.expected = promotedType;
@@ -184,8 +184,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply not equals [!=] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -234,8 +234,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         left.expected = promotedType;
@@ -275,8 +275,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -315,8 +315,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply greater than [>] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -355,8 +355,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
@@ -395,8 +395,8 @@ public final class EComp extends AExpression {
 
         if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply less than [>=] to types " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(left.actual) + "] and " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(right.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(left.actual) + "] and " +
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(right.actual) + "]."));
         }
 
         if (promotedType == def.class) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 92b14a885a1..d787db5d41c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -68,13 +68,13 @@ public final class EFunctionRef extends AExpression implements ILambda {
                     PainlessMethod interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod;
                     if (interfaceMethod == null) {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                                "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], not a functional interface");
+                                "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface");
                     }
                     PainlessMethod delegateMethod =
                             locals.getMethod(PainlessLookupUtility.buildPainlessMethodKey(call, interfaceMethod.arguments.size()));
                     if (delegateMethod == null) {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
-                                "to [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], function not found");
+                                "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], function not found");
                     }
                     ref = new FunctionRef(expected, interfaceMethod, delegateMethod, 0);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index 05564a2952e..2fa8ca8ca95 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -64,8 +64,8 @@ public final class EInstanceof extends AExpression {
         }
 
         // map to wrapped type for primitive types
-        resolvedType = clazz.isPrimitive() ? PainlessLookupUtility.getBoxedAnyType(clazz) :
-                PainlessLookupUtility.painlessDefTypeToJavaObjectType(clazz);
+        resolvedType = clazz.isPrimitive() ? PainlessLookupUtility.typeToBoxedType(clazz) :
+                PainlessLookupUtility.typeToJavaType(clazz);
 
         // analyze and cast the expression
         expression.analyze(locals);
@@ -76,7 +76,7 @@ public final class EInstanceof extends AExpression {
         primitiveExpression = expression.actual.isPrimitive();
         // map to wrapped type for primitive types
         expressionType = expression.actual.isPrimitive() ?
-            PainlessLookupUtility.getBoxedAnyType(expression.actual) : PainlessLookupUtility.painlessDefTypeToJavaObjectType(clazz);
+            PainlessLookupUtility.typeToBoxedType(expression.actual) : PainlessLookupUtility.typeToJavaType(clazz);
 
         actual = boolean.class;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index 8e8d164b03d..ab1442be805 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -123,12 +123,12 @@ public final class ELambda extends AExpression implements ILambda {
             interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod;
             if (interfaceMethod == null) {
                 throw createError(new IllegalArgumentException("Cannot pass lambda to " +
-                        "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "], not a functional interface"));
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"));
             }
             // check arity before we manipulate parameters
             if (interfaceMethod.arguments.size() != paramTypeStrs.size())
                 throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name +
-                        "] in [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "]");
+                        "] in [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]");
             // for method invocation, its allowed to ignore the return value
             if (interfaceMethod.rtn == void.class) {
                 returnType = def.class;
@@ -140,7 +140,7 @@ public final class ELambda extends AExpression implements ILambda {
             for (int i = 0; i < paramTypeStrs.size(); i++) {
                 String paramType = paramTypeStrs.get(i);
                 if (paramType == null) {
-                    actualParamTypeStrs.add(PainlessLookupUtility.anyTypeToPainlessTypeName(interfaceMethod.arguments.get(i)));
+                    actualParamTypeStrs.add(PainlessLookupUtility.typeToCanonicalTypeName(interfaceMethod.arguments.get(i)));
                 } else {
                     actualParamTypeStrs.add(paramType);
                 }
@@ -162,14 +162,14 @@ public final class ELambda extends AExpression implements ILambda {
         List<String> paramTypes = new ArrayList<>(captures.size() + actualParamTypeStrs.size());
         List<String> paramNames = new ArrayList<>(captures.size() + paramNameStrs.size());
         for (Variable var : captures) {
-            paramTypes.add(PainlessLookupUtility.anyTypeToPainlessTypeName(var.clazz));
+            paramTypes.add(PainlessLookupUtility.typeToCanonicalTypeName(var.clazz));
             paramNames.add(var.name);
         }
         paramTypes.addAll(actualParamTypeStrs);
         paramNames.addAll(paramNameStrs);
 
         // desugar lambda body into a synthetic method
-        desugared = new SFunction(reserved, location, PainlessLookupUtility.anyTypeToPainlessTypeName(returnType), name,
+        desugared = new SFunction(reserved, location, PainlessLookupUtility.typeToCanonicalTypeName(returnType), name,
                                   paramTypes, paramNames, statements, true);
         desugared.generateSignature(locals.getPainlessLookup());
         desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), returnType,
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 6bc5331cb1d..3a47dfc725f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -53,7 +53,7 @@ public final class ENull extends AExpression {
         if (expected != null) {
             if (expected.isPrimitive()) {
                 throw createError(new IllegalArgumentException(
-                    "Cannot cast null to a primitive type [" + PainlessLookupUtility.anyTypeToPainlessTypeName(expected) + "]."));
+                    "Cannot cast null to a primitive type [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]."));
             }
 
             actual = expected;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index d34399db779..1c0fce81876 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -94,7 +94,7 @@ public final class EUnary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply not [~] to type " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(child.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(child.actual) + "]."));
         }
 
         child.expected = promote;
@@ -124,7 +124,7 @@ public final class EUnary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply positive [+] to type " +
-                    "[" + PainlessLookupUtility.painlessDefTypeToJavaObjectType(child.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToJavaType(child.actual) + "]."));
         }
 
         child.expected = promote;
@@ -158,7 +158,7 @@ public final class EUnary extends AExpression {
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply negative [-] to type " +
-                    "[" + PainlessLookupUtility.painlessDefTypeToJavaObjectType(child.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToJavaType(child.actual) + "]."));
         }
 
         child.expected = promote;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index c45107a37ac..7b55cb5a804 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -68,7 +68,7 @@ public final class PBrace extends AStoreable {
             sub = new PSubListShortcut(location, locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual), index);
         } else {
             throw createError(new IllegalArgumentException("Illegal array access on type " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "]."));
         }
 
         sub.write = write;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index cd5d6483791..8fc8a612b84 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -73,7 +73,7 @@ public final class PCallInvoke extends AExpression {
         PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual);
 
         if (prefix.actual.isPrimitive()) {
-            struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(PainlessLookupUtility.getBoxedAnyType(prefix.actual));
+            struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(PainlessLookupUtility.typeToBoxedType(prefix.actual));
         }
 
         String methodKey = PainlessLookupUtility.buildPainlessMethodKey(name, arguments.size());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index b5df74358d3..abf398d0e67 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -63,7 +63,7 @@ public final class PField extends AStoreable {
         prefix = prefix.cast(locals);
 
         if (prefix.actual.isArray()) {
-            sub = new PSubArrayLength(location, PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual), value);
+            sub = new PSubArrayLength(location, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), value);
         } else if (prefix.actual == def.class) {
             sub = new PSubDefField(location, value);
         } else {
@@ -85,7 +85,8 @@ public final class PField extends AStoreable {
                         "set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
                 if (getter != null || setter != null) {
-                    sub = new PSubShortcut(location, value, PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual), getter, setter);
+                    sub = new PSubShortcut(
+                            location, value, PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual), getter, setter);
                 } else {
                     EConstant index = new EConstant(location, value);
                     index.analyze(locals);
@@ -103,7 +104,7 @@ public final class PField extends AStoreable {
 
         if (sub == null) {
             throw createError(new IllegalArgumentException(
-                "Unknown field [" + value + "] for type [" + PainlessLookupUtility.anyTypeToPainlessTypeName(prefix.actual) + "]."));
+                "Unknown field [" + value + "] for type [" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "]."));
         }
 
         if (nullSafe) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index a1a0ee1dade..007a599e9f8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -53,7 +53,7 @@ final class PSubField extends AStoreable {
     void analyze(Locals locals) {
          if (write && Modifier.isFinal(field.modifiers)) {
              throw createError(new IllegalArgumentException("Cannot write to read-only field [" + field.name + "] for type " +
-                     "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(field.clazz) + "]."));
+                     "[" + PainlessLookupUtility.typeToCanonicalTypeName(field.clazz) + "]."));
          }
 
         actual = field.clazz;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index de1a7062a24..9ff57e6b913 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -85,7 +85,7 @@ public class SEach extends AStatement {
             sub = new SSubEachIterable(location, variable, expression, block);
         } else {
             throw createError(new IllegalArgumentException("Illegal for each type " +
-                    "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(expression.actual) + "]."));
+                    "[" + PainlessLookupUtility.typeToCanonicalTypeName(expression.actual) + "]."));
         }
 
         sub.analyze(locals);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 1c801d509b5..7c243e296c7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -136,7 +136,7 @@ public final class SFunction extends AStatement {
             try {
                 Class<?> paramType = painlessLookup.getJavaClassFromPainlessType(this.paramTypeStrs.get(param));
 
-                paramClasses[param] = PainlessLookupUtility.painlessDefTypeToJavaObjectType(paramType);
+                paramClasses[param] = PainlessLookupUtility.typeToJavaType(paramType);
                 paramTypes.add(paramType);
                 parameters.add(new Parameter(location, paramNameStrs.get(param), paramType));
             } catch (IllegalArgumentException exception) {
@@ -146,7 +146,7 @@ public final class SFunction extends AStatement {
         }
 
         org.objectweb.asm.commons.Method method = new org.objectweb.asm.commons.Method(name, MethodType.methodType(
-                PainlessLookupUtility.painlessDefTypeToJavaObjectType(rtnType), paramClasses).toMethodDescriptorString());
+                PainlessLookupUtility.typeToJavaType(rtnType), paramClasses).toMethodDescriptorString());
         this.method = new PainlessMethod(name, null, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index fea8c8953b6..7e0d74865f9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -109,6 +109,6 @@ final class SSubEachArray extends AStatement {
 
     @Override
     public String toString() {
-        return singleLineToString(PainlessLookupUtility.anyTypeToPainlessTypeName(variable.clazz), variable.name, expression, block);
+        return singleLineToString(PainlessLookupUtility.typeToCanonicalTypeName(variable.clazz), variable.name, expression, block);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 798b30e2b6d..12e3154eb56 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -81,7 +81,7 @@ final class SSubEachIterable extends AStatement {
 
             if (method == null) {
                 throw createError(new IllegalArgumentException("Unable to create iterator for the type " +
-                        "[" + PainlessLookupUtility.anyTypeToPainlessTypeName(expression.actual) + "]."));
+                        "[" + PainlessLookupUtility.typeToCanonicalTypeName(expression.actual) + "]."));
             }
         }
 
@@ -132,6 +132,6 @@ final class SSubEachIterable extends AStatement {
 
     @Override
     public String toString() {
-        return singleLineToString(PainlessLookupUtility.anyTypeToPainlessTypeName(variable.clazz), variable.name, expression, block);
+        return singleLineToString(PainlessLookupUtility.typeToCanonicalTypeName(variable.clazz), variable.name, expression, block);
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index cc596dcc395..e26a5a38c76 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -210,7 +210,7 @@ public class PainlessDocGenerator {
      */
     private static void emitAnchor(PrintStream stream, Class<?> clazz) {
         stream.print("painless-api-reference-");
-        stream.print(PainlessLookupUtility.anyTypeToPainlessTypeName(clazz).replace('.', '-'));
+        stream.print(PainlessLookupUtility.typeToCanonicalTypeName(clazz).replace('.', '-'));
     }
 
     /**
@@ -234,7 +234,7 @@ public class PainlessDocGenerator {
     }
 
     private static String methodName(PainlessMethod method) {
-        return method.name.equals("<init>") ? PainlessLookupUtility.anyTypeToPainlessTypeName(method.target) : method.name;
+        return method.name.equals("<init>") ? PainlessLookupUtility.typeToCanonicalTypeName(method.target) : method.name;
     }
 
     /**

From c32981db6bc4b04383a801ba40c6781d22f262fe Mon Sep 17 00:00:00 2001
From: Tim Vernum <tim@adjective.org>
Date: Fri, 20 Jul 2018 10:09:28 +1000
Subject: [PATCH 106/260] Detect old trial licenses and mimic behaviour
 (#32209)

Prior to 6.3 a trial license default to security enabled. Since 6.3
they default to security disabled. If a cluster is upgraded from <6.3
to >6.3, then we detect this and mimic the old behaviour with respect
to security.
---
 .../elasticsearch/license/LicenseService.java | 25 ++++++--
 .../license/XPackLicenseState.java            | 29 ++++++++-
 .../org/elasticsearch/license/TestUtils.java  | 11 ++--
 .../license/XPackLicenseStateTests.java       | 64 ++++++++++++++-----
 .../MachineLearningLicensingTests.java        |  8 +--
 .../elasticsearch/license/LicensingTests.java |  4 +-
 .../xpack/security/SecurityTests.java         |  3 +-
 .../authz/store/CompositeRolesStoreTests.java |  6 +-
 .../action/saml/SamlBaseRestHandlerTests.java |  4 +-
 9 files changed, 111 insertions(+), 43 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
index a39e9f412d7..d5e38fc0cb8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
@@ -120,7 +120,8 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
         this.scheduler = new SchedulerEngine(clock);
         this.licenseState = licenseState;
         this.operationModeFileWatcher = new OperationModeFileWatcher(resourceWatcherService,
-                XPackPlugin.resolveConfigFile(env, "license_mode"), logger, () -> updateLicenseState(getLicense()));
+            XPackPlugin.resolveConfigFile(env, "license_mode"), logger,
+            () -> updateLicenseState(getLicensesMetaData()));
         this.scheduler.register(this);
         populateExpirationCallbacks();
     }
@@ -265,11 +266,11 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
 
     @Override
     public void triggered(SchedulerEngine.Event event) {
-        final LicensesMetaData licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE);
+        final LicensesMetaData licensesMetaData = getLicensesMetaData();
         if (licensesMetaData != null) {
             final License license = licensesMetaData.getLicense();
             if (event.getJobName().equals(LICENSE_JOB)) {
-                updateLicenseState(license);
+                updateLicenseState(license, licensesMetaData.getMostRecentTrialVersion());
             } else if (event.getJobName().startsWith(ExpirationCallback.EXPIRATION_JOB_PREFIX)) {
                 expirationCallbacks.stream()
                         .filter(expirationCallback -> expirationCallback.getId().equals(event.getJobName()))
@@ -311,6 +312,10 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
         return license == LicensesMetaData.LICENSE_TOMBSTONE ? null : license;
     }
 
+    private LicensesMetaData getLicensesMetaData() {
+        return this.clusterService.state().metaData().custom(LicensesMetaData.TYPE);
+    }
+
     void startTrialLicense(PostStartTrialRequest request, final ActionListener<PostStartTrialResponse> listener) {
         if (VALID_TRIAL_TYPES.contains(request.getType()) == false) {
             throw new IllegalArgumentException("Cannot start trial of type [" + request.getType() + "]. Valid trial types are "
@@ -422,10 +427,16 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
         }
     }
 
-    protected void updateLicenseState(final License license) {
+    private void updateLicenseState(LicensesMetaData licensesMetaData) {
+        if (licensesMetaData != null) {
+            updateLicenseState(getLicense(licensesMetaData), licensesMetaData.getMostRecentTrialVersion());
+        }
+    }
+
+    protected void updateLicenseState(final License license, Version mostRecentTrialVersion) {
         if (license == LicensesMetaData.LICENSE_TOMBSTONE) {
             // implies license has been explicitly deleted
-            licenseState.update(License.OperationMode.MISSING, false);
+            licenseState.update(License.OperationMode.MISSING, false, mostRecentTrialVersion);
             return;
         }
         if (license != null) {
@@ -438,7 +449,7 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
                 // date that is near Long.MAX_VALUE
                 active = time >= license.issueDate() && time - GRACE_PERIOD_DURATION.getMillis() < license.expiryDate();
             }
-            licenseState.update(license.operationMode(), active);
+            licenseState.update(license.operationMode(), active, mostRecentTrialVersion);
 
             if (active) {
                 if (time < license.expiryDate()) {
@@ -480,7 +491,7 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
                 logger.info("license [{}] mode [{}] - valid", license.uid(),
                         license.operationMode().name().toLowerCase(Locale.ROOT));
             }
-            updateLicenseState(license);
+            updateLicenseState(license, currentLicensesMetaData.getMostRecentTrialVersion());
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
index ea30e30ae3c..722c9d0e711 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -5,8 +5,11 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.License.OperationMode;
 import org.elasticsearch.xpack.core.XPackField;
@@ -266,6 +269,7 @@ public class XPackLicenseState {
     private final List<Runnable> listeners = new CopyOnWriteArrayList<>();
     private final boolean isSecurityEnabled;
     private final boolean isSecurityExplicitlyEnabled;
+    private volatile boolean isSecurityEnabledByTrialVersion;
 
     public XPackLicenseState(Settings settings) {
         this.isSecurityEnabled = XPackSettings.SECURITY_ENABLED.get(settings);
@@ -274,11 +278,30 @@ public class XPackLicenseState {
         // setting is not explicitly set
         this.isSecurityExplicitlyEnabled = isSecurityEnabled &&
             (settings.hasValue(XPackSettings.SECURITY_ENABLED.getKey()) || XPackSettings.TRANSPORT_SSL_ENABLED.get(settings));
+        this.isSecurityEnabledByTrialVersion = false;
     }
 
-    /** Updates the current state of the license, which will change what features are available. */
-    void update(OperationMode mode, boolean active) {
+    /**
+     * Updates the current state of the license, which will change what features are available.
+     *
+     * @param mode   The mode (type) of the current license.
+     * @param active True if the current license exists and is within its allowed usage period; false if it is expired or missing.
+     * @param mostRecentTrialVersion If this cluster has, at some point commenced a trial, the most recent version on which they did that.
+     *                               May be {@code null} if they have never generated a trial license on this cluster, or the most recent
+     *                               trial was prior to this metadata being tracked (6.1)
+     */
+    void update(OperationMode mode, boolean active, @Nullable Version mostRecentTrialVersion) {
         status = new Status(mode, active);
+        if (isSecurityEnabled == true && isSecurityExplicitlyEnabled == false && mode == OperationMode.TRIAL
+            && isSecurityEnabledByTrialVersion == false) {
+            // Before 6.3, Trial licenses would default having security enabled.
+            // If this license was generated before that version, then treat it as if security is explicitly enabled
+            if (mostRecentTrialVersion == null || mostRecentTrialVersion.before(Version.V_6_3_0)) {
+                Loggers.getLogger(getClass()).info("Automatically enabling security for older trial license ({})",
+                    mostRecentTrialVersion == null ? "[pre 6.1.0]" : mostRecentTrialVersion.toString());
+                isSecurityEnabledByTrialVersion = true;
+            }
+        }
         listeners.forEach(Runnable::run);
     }
 
@@ -587,6 +610,6 @@ public class XPackLicenseState {
 
     public boolean isSecurityEnabled() {
         final OperationMode mode = status.mode;
-        return mode == OperationMode.TRIAL ? isSecurityExplicitlyEnabled : isSecurityEnabled;
+        return mode == OperationMode.TRIAL ? (isSecurityExplicitlyEnabled || isSecurityEnabledByTrialVersion) : isSecurityEnabled;
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
index d236dacaa4d..3b7906ae567 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.license;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
@@ -353,20 +354,22 @@ public class TestUtils {
     public static class AssertingLicenseState extends XPackLicenseState {
         public final List<License.OperationMode> modeUpdates = new ArrayList<>();
         public final List<Boolean> activeUpdates = new ArrayList<>();
+        public final List<Version> trialVersionUpdates = new ArrayList<>();
 
         public AssertingLicenseState() {
             super(Settings.EMPTY);
         }
 
         @Override
-        void update(License.OperationMode mode, boolean active) {
+        void update(License.OperationMode mode, boolean active, Version mostRecentTrialVersion) {
             modeUpdates.add(mode);
             activeUpdates.add(active);
+            trialVersionUpdates.add(mostRecentTrialVersion);
         }
     }
 
     /**
-     * A license state that makes the {@link #update(License.OperationMode, boolean)}
+     * A license state that makes the {@link #update(License.OperationMode, boolean, Version)}
      * method public for use in tests.
      */
     public static class UpdatableLicenseState extends XPackLicenseState {
@@ -379,8 +382,8 @@ public class TestUtils {
         }
 
         @Override
-        public void update(License.OperationMode mode, boolean active) {
-            super.update(mode, active);
+        public void update(License.OperationMode mode, boolean active, Version mostRecentTrialVersion) {
+            super.update(mode, active, mostRecentTrialVersion);
         }
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
index f1503919570..bb21ddbd1a1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
@@ -5,9 +5,11 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.License.OperationMode;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
 
@@ -31,7 +33,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     /** Creates a license state with the given license type and active state, and checks the given method returns expected. */
     void assertAllowed(OperationMode mode, boolean active, Predicate<XPackLicenseState> predicate, boolean expected) {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(mode, active);
+        licenseState.update(mode, active, null);
         assertEquals(expected, predicate.test(licenseState));
     }
 
@@ -102,7 +104,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     public void testSecurityBasic() {
         XPackLicenseState licenseState = new XPackLicenseState(randomFrom(Settings.EMPTY,
                 Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build()));
-        licenseState.update(BASIC, true);
+        licenseState.update(BASIC, true, null);
 
         assertThat(licenseState.isAuthAllowed(), is(false));
         assertThat(licenseState.isIpFilteringAllowed(), is(false));
@@ -116,7 +118,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     public void testSecurityBasicExpired() {
         XPackLicenseState licenseState = new XPackLicenseState(randomFrom(Settings.EMPTY,
                 Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build()));
-        licenseState.update(BASIC, false);
+        licenseState.update(BASIC, false, null);
 
         assertThat(licenseState.isAuthAllowed(), is(false));
         assertThat(licenseState.isIpFilteringAllowed(), is(false));
@@ -130,7 +132,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     public void testSecurityStandard() {
         XPackLicenseState licenseState = new XPackLicenseState(randomFrom(Settings.EMPTY,
                 Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build()));
-        licenseState.update(STANDARD, true);
+        licenseState.update(STANDARD, true, null);
 
         assertThat(licenseState.isAuthAllowed(), is(true));
         assertThat(licenseState.isIpFilteringAllowed(), is(false));
@@ -144,7 +146,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     public void testSecurityStandardExpired() {
         XPackLicenseState licenseState = new XPackLicenseState(randomFrom(Settings.EMPTY,
                 Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build()));
-        licenseState.update(STANDARD, false);
+        licenseState.update(STANDARD, false, null);
 
         assertThat(licenseState.isAuthAllowed(), is(true));
         assertThat(licenseState.isIpFilteringAllowed(), is(false));
@@ -158,7 +160,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     public void testSecurityGold() {
         XPackLicenseState licenseState = new XPackLicenseState(randomFrom(Settings.EMPTY,
                 Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build()));
-        licenseState.update(GOLD, true);
+        licenseState.update(GOLD, true, null);
 
         assertThat(licenseState.isAuthAllowed(), is(true));
         assertThat(licenseState.isIpFilteringAllowed(), is(true));
@@ -172,7 +174,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     public void testSecurityGoldExpired() {
         XPackLicenseState licenseState = new XPackLicenseState(randomFrom(Settings.EMPTY,
                 Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build()));
-        licenseState.update(GOLD, false);
+        licenseState.update(GOLD, false, null);
 
         assertThat(licenseState.isAuthAllowed(), is(true));
         assertThat(licenseState.isIpFilteringAllowed(), is(true));
@@ -186,7 +188,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     public void testSecurityPlatinum() {
         XPackLicenseState licenseState = new XPackLicenseState(randomFrom(Settings.EMPTY,
                 Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build()));
-        licenseState.update(PLATINUM, true);
+        licenseState.update(PLATINUM, true, null);
 
         assertThat(licenseState.isAuthAllowed(), is(true));
         assertThat(licenseState.isIpFilteringAllowed(), is(true));
@@ -200,7 +202,7 @@ public class XPackLicenseStateTests extends ESTestCase {
     public void testSecurityPlatinumExpired() {
         XPackLicenseState licenseState = new XPackLicenseState(randomFrom(Settings.EMPTY,
                 Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build()));
-        licenseState.update(PLATINUM, false);
+        licenseState.update(PLATINUM, false, null);
 
         assertThat(licenseState.isAuthAllowed(), is(true));
         assertThat(licenseState.isIpFilteringAllowed(), is(true));
@@ -211,6 +213,34 @@ public class XPackLicenseStateTests extends ESTestCase {
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
     }
 
+    public void testNewTrialDefaultsSecurityOff() {
+        XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
+        licenseState.update(TRIAL, true, VersionUtils.randomVersionBetween(random(), Version.V_6_3_0, Version.CURRENT));
+
+        assertThat(licenseState.isSecurityEnabled(), is(false));
+        assertThat(licenseState.isAuthAllowed(), is(true));
+        assertThat(licenseState.isIpFilteringAllowed(), is(true));
+        assertThat(licenseState.isAuditingAllowed(), is(true));
+        assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
+        assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(true));
+        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.ALL));
+        assertThat(licenseState.isCustomRoleProvidersAllowed(), is(true));
+    }
+
+    public void testOldTrialDefaultsSecurityOn() {
+        XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
+        licenseState.update(TRIAL, true, rarely() ? null : VersionUtils.randomVersionBetween(random(), Version.V_5_6_0, Version.V_6_2_4));
+
+        assertThat(licenseState.isSecurityEnabled(), is(true));
+        assertThat(licenseState.isAuthAllowed(), is(true));
+        assertThat(licenseState.isIpFilteringAllowed(), is(true));
+        assertThat(licenseState.isAuditingAllowed(), is(true));
+        assertThat(licenseState.isStatsAndHealthAllowed(), is(true));
+        assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(true));
+        assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.ALL));
+        assertThat(licenseState.isCustomRoleProvidersAllowed(), is(true));
+    }
+
     public void testSecurityAckBasicToNotGoldOrStandard() {
         OperationMode toMode = randomFrom(OperationMode.values(), mode -> mode != GOLD && mode != STANDARD);
         assertAckMesssages(XPackField.SECURITY, BASIC, toMode, 0);
@@ -354,7 +384,7 @@ public class XPackLicenseStateTests extends ESTestCase {
 
     public void testSqlBasic() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(BASIC, true);
+        licenseState.update(BASIC, true, null);
 
         assertThat(licenseState.isSqlAllowed(), is(true));
         assertThat(licenseState.isJdbcAllowed(), is(false));
@@ -362,7 +392,7 @@ public class XPackLicenseStateTests extends ESTestCase {
 
     public void testSqlBasicExpired() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(BASIC, false);
+        licenseState.update(BASIC, false, null);
 
         assertThat(licenseState.isSqlAllowed(), is(false));
         assertThat(licenseState.isJdbcAllowed(), is(false));
@@ -370,7 +400,7 @@ public class XPackLicenseStateTests extends ESTestCase {
 
     public void testSqlStandard() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(STANDARD, true);
+        licenseState.update(STANDARD, true, null);
 
         assertThat(licenseState.isSqlAllowed(), is(true));
         assertThat(licenseState.isJdbcAllowed(), is(false));
@@ -378,7 +408,7 @@ public class XPackLicenseStateTests extends ESTestCase {
 
     public void testSqlStandardExpired() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(STANDARD, false);
+        licenseState.update(STANDARD, false, null);
 
         assertThat(licenseState.isSqlAllowed(), is(false));
         assertThat(licenseState.isJdbcAllowed(), is(false));
@@ -386,7 +416,7 @@ public class XPackLicenseStateTests extends ESTestCase {
 
     public void testSqlGold() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(GOLD, true);
+        licenseState.update(GOLD, true, null);
 
         assertThat(licenseState.isSqlAllowed(), is(true));
         assertThat(licenseState.isJdbcAllowed(), is(false));
@@ -394,7 +424,7 @@ public class XPackLicenseStateTests extends ESTestCase {
 
     public void testSqlGoldExpired() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(GOLD, false);
+        licenseState.update(GOLD, false, null);
 
         assertThat(licenseState.isSqlAllowed(), is(false));
         assertThat(licenseState.isJdbcAllowed(), is(false));
@@ -402,7 +432,7 @@ public class XPackLicenseStateTests extends ESTestCase {
 
     public void testSqlPlatinum() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(PLATINUM, true);
+        licenseState.update(PLATINUM, true, null);
 
         assertThat(licenseState.isSqlAllowed(), is(true));
         assertThat(licenseState.isJdbcAllowed(), is(true));
@@ -410,7 +440,7 @@ public class XPackLicenseStateTests extends ESTestCase {
 
     public void testSqlPlatinumExpired() {
         XPackLicenseState licenseState = new XPackLicenseState(Settings.EMPTY);
-        licenseState.update(PLATINUM, false);
+        licenseState.update(PLATINUM, false, null);
 
         assertThat(licenseState.isSqlAllowed(), is(false));
         assertThat(licenseState.isJdbcAllowed(), is(false));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
index 558f76a7536..57a97641791 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
@@ -99,7 +99,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
             PutJobAction.Response response = putJobListener.actionGet();
             assertNotNull(response);
         }
-        
+
         // Pick a license that does not allow machine learning
         License.OperationMode mode = randomInvalidLicenseType();
         enableLicensing(mode);
@@ -151,7 +151,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
             PutJobAction.Response putJobResponse = putJobListener.actionGet();
             assertNotNull(putJobResponse);
         }
-        
+
         // Pick a license that does not allow machine learning
         License.OperationMode mode = randomInvalidLicenseType();
         enableLicensing(mode);
@@ -551,7 +551,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
 
     public static void disableLicensing(License.OperationMode operationMode) {
         for (XPackLicenseState licenseState : internalCluster().getInstances(XPackLicenseState.class)) {
-            licenseState.update(operationMode, false);
+            licenseState.update(operationMode, false, null);
         }
     }
 
@@ -561,7 +561,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
 
     public static void enableLicensing(License.OperationMode operationMode) {
         for (XPackLicenseState licenseState : internalCluster().getInstances(XPackLicenseState.class)) {
-            licenseState.update(operationMode, true);
+            licenseState.update(operationMode, true, null);
         }
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
index f30fe5f8ec8..012050f4259 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
@@ -307,7 +307,7 @@ public class LicensingTests extends SecurityIntegTestCase {
 
     public static void disableLicensing(License.OperationMode operationMode) {
         for (XPackLicenseState licenseState : internalCluster().getInstances(XPackLicenseState.class)) {
-            licenseState.update(operationMode, false);
+            licenseState.update(operationMode, false, null);
         }
     }
 
@@ -317,7 +317,7 @@ public class LicensingTests extends SecurityIntegTestCase {
 
     public static void enableLicensing(License.OperationMode operationMode) {
         for (XPackLicenseState licenseState : internalCluster().getInstances(XPackLicenseState.class)) {
-            licenseState.update(operationMode, true);
+            licenseState.update(operationMode, true, null);
         }
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index b1d8d4b67bf..e88b1905a7a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -399,7 +399,8 @@ public class SecurityTests extends ESTestCase {
         createComponents(Settings.EMPTY);
         Function<String, Predicate<String>> fieldFilter = security.getFieldFilter();
         assertNotSame(MapperPlugin.NOOP_FIELD_FILTER, fieldFilter);
-        licenseState.update(randomFrom(License.OperationMode.BASIC, License.OperationMode.STANDARD, License.OperationMode.GOLD), true);
+        licenseState.update(
+            randomFrom(License.OperationMode.BASIC, License.OperationMode.STANDARD, License.OperationMode.GOLD), true, null);
         assertNotSame(MapperPlugin.NOOP_FIELD_FILTER, fieldFilter);
         assertSame(MapperPlugin.NOOP_FIELD_PREDICATE, fieldFilter.apply(randomAlphaOfLengthBetween(3, 6)));
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
index b1728fd5f04..b33e93d0806 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
@@ -409,7 +409,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
 
         UpdatableLicenseState xPackLicenseState = new UpdatableLicenseState(SECURITY_ENABLED_SETTINGS);
         // these licenses don't allow custom role providers
-        xPackLicenseState.update(randomFrom(OperationMode.BASIC, OperationMode.GOLD, OperationMode.STANDARD), true);
+        xPackLicenseState.update(randomFrom(OperationMode.BASIC, OperationMode.GOLD, OperationMode.STANDARD), true, null);
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState);
@@ -427,7 +427,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState);
         // these licenses allow custom role providers
-        xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), true);
+        xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), true, null);
         roleNames = Sets.newHashSet("roleA");
         future = new PlainActionFuture<>();
         fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
@@ -441,7 +441,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState);
-        xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), false);
+        xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), false, null);
         roleNames = Sets.newHashSet("roleA");
         future = new PlainActionFuture<>();
         fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/saml/SamlBaseRestHandlerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/saml/SamlBaseRestHandlerTests.java
index 5942c206cac..5b442deacf6 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/saml/SamlBaseRestHandlerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/saml/SamlBaseRestHandlerTests.java
@@ -48,7 +48,7 @@ public class SamlBaseRestHandlerTests extends ESTestCase {
                 .put(XPackSettings.SECURITY_ENABLED.getKey(), true)
                 .build();
         final TestUtils.UpdatableLicenseState licenseState = new TestUtils.UpdatableLicenseState(settings);
-        licenseState.update(licenseMode, true);
+        licenseState.update(licenseMode, true, null);
 
         return new SamlBaseRestHandler(settings, licenseState) {
 
@@ -64,4 +64,4 @@ public class SamlBaseRestHandlerTests extends ESTestCase {
         };
     }
 
-}
\ No newline at end of file
+}

From 6f2b7dc9fe6143d2cc2bf26d83983a3a49c40b90 Mon Sep 17 00:00:00 2001
From: Tim Vernum <tim.vernum@elastic.co>
Date: Fri, 20 Jul 2018 12:51:52 +1000
Subject: [PATCH 107/260] Fix AD / vagrant based tests for #30953

These tests were creating a SSL service that was not aware of the
realm that they were trying to test.
This no longer works.
---
 .../ADLdapUserSearchSessionFactoryTests.java  |  2 +-
 .../ActiveDirectorySessionFactoryTests.java   | 50 +++++++++----------
 2 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java
index 32452a609e2..9f97ebc6d03 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java
@@ -85,7 +85,7 @@ public class ADLdapUserSearchSessionFactoryTests extends AbstractActiveDirectory
         Settings.Builder builder = Settings.builder()
                 .put(globalSettings);
         settings.keySet().forEach(k -> {
-            builder.copy("xpack.security.authc.realms.ldap." + k, k, settings);
+            builder.copy("xpack.security.authc.realms.ad-as-ldap-test." + k, k, settings);
 
         });
         Settings fullSettings = builder.build();
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
index 614d6659f2d..ec4e8824a19 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
@@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -59,9 +60,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
 
     @SuppressWarnings("unchecked")
     public void testAdAuth() throws Exception {
-        RealmConfig config = new RealmConfig("ad-test",
-                buildAdSettings(AD_LDAP_URL, AD_DOMAIN, false),
-                globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(Settings.EMPTY));
+        RealmConfig config = configureRealm("ad-test", buildAdSettings(AD_LDAP_URL, AD_DOMAIN, false));
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             String userName = "ironman";
@@ -82,11 +81,21 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
         }
     }
 
+    private RealmConfig configureRealm(String name, Settings settings) {
+        final Environment env = TestEnvironment.newEnvironment(globalSettings);
+        final Settings mergedSettings = Settings.builder()
+            .put(settings)
+            .normalizePrefix("xpack.security.authc.realms." + name + ".")
+            .put(globalSettings)
+            .build();
+        this.sslService = new SSLService(mergedSettings, env);
+        return new RealmConfig(name, settings, globalSettings, env, new ThreadContext(globalSettings));
+    }
+
     @SuppressWarnings("unchecked")
     public void testNetbiosAuth() throws Exception {
         final String adUrl = randomFrom(AD_LDAP_URL, AD_LDAP_GC_URL);
-        RealmConfig config = new RealmConfig("ad-test", buildAdSettings(adUrl, AD_DOMAIN, false), globalSettings,
-                TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-test", buildAdSettings(adUrl, AD_DOMAIN, false));
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             String userName = "ades\\ironman";
@@ -108,8 +117,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
     }
 
     public void testAdAuthAvengers() throws Exception {
-        RealmConfig config = new RealmConfig("ad-test", buildAdSettings(AD_LDAP_URL, AD_DOMAIN, false), globalSettings,
-                TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-test", buildAdSettings(AD_LDAP_URL, AD_DOMAIN, false));
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             String[] users = new String[]{"cap", "hawkeye", "hulk", "ironman", "thor", "blackwidow"};
@@ -126,8 +134,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
     public void testAuthenticate() throws Exception {
         Settings settings = buildAdSettings(AD_LDAP_URL, AD_DOMAIN, "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
                 LdapSearchScope.ONE_LEVEL, false);
-        RealmConfig config = new RealmConfig("ad-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
-                new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-test", settings);
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             String userName = "hulk";
@@ -151,8 +158,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
     public void testAuthenticateBaseUserSearch() throws Exception {
         Settings settings = buildAdSettings(AD_LDAP_URL, AD_DOMAIN, "CN=Bruce Banner, CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
                 LdapSearchScope.BASE, false);
-        RealmConfig config = new RealmConfig("ad-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
-                new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-test", settings);
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             String userName = "hulk";
@@ -180,8 +186,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
                         "CN=Avengers,CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com")
                 .put(ActiveDirectorySessionFactorySettings.AD_GROUP_SEARCH_SCOPE_SETTING, LdapSearchScope.BASE)
                 .build();
-        RealmConfig config = new RealmConfig("ad-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
-                new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-test", settings);
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             String userName = "hulk";
@@ -198,8 +203,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
     public void testAuthenticateWithUserPrincipalName() throws Exception {
         Settings settings = buildAdSettings(AD_LDAP_URL, AD_DOMAIN, "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
                 LdapSearchScope.ONE_LEVEL, false);
-        RealmConfig config = new RealmConfig("ad-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
-                new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-test", settings);
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             //Login with the UserPrincipalName
@@ -220,8 +224,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
     public void testAuthenticateWithSAMAccountName() throws Exception {
         Settings settings = buildAdSettings(AD_LDAP_URL, AD_DOMAIN, "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
                 LdapSearchScope.ONE_LEVEL, false);
-        RealmConfig config = new RealmConfig("ad-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
-                new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-test", settings);
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             //login with sAMAccountName
@@ -247,8 +250,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
                 .put(ActiveDirectorySessionFactorySettings.AD_USER_SEARCH_FILTER_SETTING,
                         "(&(objectclass=user)(userPrincipalName={0}@ad.test.elasticsearch.com))")
                 .build();
-        RealmConfig config = new RealmConfig("ad-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
-                new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-test", settings);
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             //Login with the UserPrincipalName
@@ -284,8 +286,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
                 .putList("ssl.certificate_authorities", certificatePaths)
                 .build();
         }
-        RealmConfig config = new RealmConfig("ad-as-ldap-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
-                new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-as-ldap-test", settings);
         LdapSessionFactory sessionFactory = new LdapSessionFactory(config, sslService, threadPool);
 
         String user = "Bruce Banner";
@@ -348,8 +349,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
                 .putList("ssl.certificate_authorities", certificatePaths)
                 .build();
         }
-        RealmConfig config = new RealmConfig("ad-as-ldap-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
-                new ThreadContext(globalSettings));
+        RealmConfig config = configureRealm("ad-as-ldap-test", settings);
         LdapSessionFactory sessionFactory = new LdapSessionFactory(config, sslService, threadPool);
 
         String user = "Bruce Banner";
@@ -366,9 +366,7 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
     }
 
     public void testADLookup() throws Exception {
-        RealmConfig config = new RealmConfig("ad-test",
-                buildAdSettings(AD_LDAP_URL, AD_DOMAIN, false, true),
-                globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(Settings.EMPTY));
+        RealmConfig config = configureRealm("ad-test", buildAdSettings(AD_LDAP_URL, AD_DOMAIN, false, true));
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
 
             List<String> users = randomSubsetOf(Arrays.asList("cap", "hawkeye", "hulk", "ironman", "thor", "blackwidow",

From a9a9598e02ac6edb27bd4940196a2e27a78d7288 Mon Sep 17 00:00:00 2001
From: Alpar Torok <torokalpar@gmail.com>
Date: Fri, 20 Jul 2018 05:20:35 +0000
Subject: [PATCH 108/260] Require Gradle 4.9  as minimum version (#32200)

Do the check in the build plugin as well to be more informative to users
of  build-tools.

We use APIs specific to Gradle 4.9 so earlier will not work.
---
 buildSrc/build.gradle                                         | 4 ++--
 .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy   | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index eb95ff148f6..55520728812 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -25,8 +25,8 @@ plugins {
 
 group = 'org.elasticsearch.gradle'
 
-if (GradleVersion.current() < GradleVersion.version('3.3')) {
-  throw new GradleException('Gradle 3.3+ is required to build elasticsearch')
+if (GradleVersion.current() < GradleVersion.version('4.9')) {
+  throw new GradleException('Gradle 4.9+ is required to build elasticsearch')
 }
 
 if (JavaVersion.current() < JavaVersion.VERSION_1_8) {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index c5dd19de3cc..9f54ae8b682 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -67,6 +67,9 @@ class BuildPlugin implements Plugin<Project> {
                 + 'elasticearch.standalone-rest-test, and elasticsearch.build '
                 + 'are mutually exclusive')
         }
+        if (GradleVersion.current() < GradleVersion.version('4.9')) {
+            throw new GradleException('Gradle 4.9+ is required to use elasticsearch.build plugin')
+        }
         project.pluginManager.apply('java')
         project.pluginManager.apply('carrotsearch.randomized-testing')
         // these plugins add lots of info to our jars

From 24068a773dcd5725ff72c1a55f3ab4bd4a0df576 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 20 Jul 2018 09:12:32 +0200
Subject: [PATCH 109/260] TESTS: Check for Netty resource leaks (#31861)

* Enabled advanced leak detection when loading `EsTestCase`
* Added custom `Appender` to collect leak logs and check for logged errors in a way similar to what is done for the `StatusLogger`
* Fixes #20398
---
 .../org/elasticsearch/test/ESTestCase.java    | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 4c00527a932..5d555ece438 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -32,8 +32,12 @@ import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LogEvent;
 import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
 import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.core.layout.PatternLayout;
 import org.apache.logging.log4j.status.StatusConsoleListener;
 import org.apache.logging.log4j.status.StatusData;
 import org.apache.logging.log4j.status.StatusLogger;
@@ -183,6 +187,8 @@ public abstract class ESTestCase extends LuceneTestCase {
 
     private static final AtomicInteger portGenerator = new AtomicInteger();
 
+    private static final Collection<String> nettyLoggedLeaks = new ArrayList<>();
+
     @AfterClass
     public static void resetPortCounter() {
         portGenerator.set(0);
@@ -192,8 +198,28 @@ public abstract class ESTestCase extends LuceneTestCase {
         System.setProperty("log4j.shutdownHookEnabled", "false");
         System.setProperty("log4j2.disable.jmx", "true");
 
+        // Enable Netty leak detection and monitor logger for logged leak errors
+        System.setProperty("io.netty.leakDetection.level", "advanced");
+        String leakLoggerName = "io.netty.util.ResourceLeakDetector";
+        Logger leakLogger = LogManager.getLogger(leakLoggerName);
+        Appender leakAppender = new AbstractAppender(leakLoggerName, null,
+            PatternLayout.newBuilder().withPattern("%m").build()) {
+            @Override
+            public void append(LogEvent event) {
+                String message = event.getMessage().getFormattedMessage();
+                if (Level.ERROR.equals(event.getLevel()) && message.contains("LEAK:")) {
+                    synchronized (nettyLoggedLeaks) {
+                        nettyLoggedLeaks.add(message);
+                    }
+                }
+            }
+        };
+        leakAppender.start();
+        Loggers.addAppender(leakLogger, leakAppender);
+
         // shutdown hook so that when the test JVM exits, logging is shutdown too
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            leakAppender.stop();
             LoggerContext context = (LoggerContext) LogManager.getContext(false);
             Configurator.shutdown(context);
         }));
@@ -440,6 +466,13 @@ public abstract class ESTestCase extends LuceneTestCase {
                 statusData.clear();
             }
         }
+        synchronized (nettyLoggedLeaks) {
+            try {
+                assertThat(nettyLoggedLeaks, empty());
+            } finally {
+                nettyLoggedLeaks.clear();
+            }
+        }
     }
 
     // this must be a separate method from other ensure checks above so suite scoped integ tests can call...TODO: fix that

From 0f0068b91cad02fe504e2ecc1a23742275120758 Mon Sep 17 00:00:00 2001
From: Julie Tibshirani <julie.tibshirani@elastic.co>
Date: Fri, 20 Jul 2018 00:18:54 -0700
Subject: [PATCH 110/260] Ensure that field aliases cannot be used in
 multi-fields. (#32219)

---
 docs/reference/mapping/types/alias.asciidoc   |  2 +-
 .../index/mapper/TypeParsers.java             |  4 +++-
 .../mapper/DocumentMapperParserTests.java     | 24 +++++++++++++++++++
 3 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/docs/reference/mapping/types/alias.asciidoc b/docs/reference/mapping/types/alias.asciidoc
index d2b5ccdce8a..b4243d80e6d 100644
--- a/docs/reference/mapping/types/alias.asciidoc
+++ b/docs/reference/mapping/types/alias.asciidoc
@@ -74,7 +74,7 @@ field alias to query over multiple target fields in a single clause.
 ==== Unsupported APIs
 
 Writes to field aliases are not supported: attempting to use an alias in an index or update request
-will result in a failure. Likewise, aliases cannot be used as the target of `copy_to`.
+will result in a failure. Likewise, aliases cannot be used as the target of `copy_to` or in multi-fields.
 
 Because alias names are not present in the document source, aliases cannot be used when performing
 source filtering. For example, the following request will return an empty result for `_source`:
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
index 52dfadfe273..a6a5fab0d04 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
@@ -230,7 +230,9 @@ public class TypeParsers {
                 } else {
                     throw new MapperParsingException("no type specified for property [" + multiFieldName + "]");
                 }
-                if (type.equals(ObjectMapper.CONTENT_TYPE) || type.equals(ObjectMapper.NESTED_CONTENT_TYPE)) {
+                if (type.equals(ObjectMapper.CONTENT_TYPE)
+                        || type.equals(ObjectMapper.NESTED_CONTENT_TYPE)
+                        || type.equals(FieldAliasMapper.CONTENT_TYPE)) {
                     throw new MapperParsingException("Type [" + type + "] cannot be used in multi field");
                 }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java
index 268b03d046c..8e164c86ebe 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java
@@ -76,4 +76,28 @@ public class DocumentMapperParserTests extends ESSingleNodeTestCase {
             mapperParser.parse("type", new CompressedXContent(mapping)));
         assertTrue(e.getMessage(), e.getMessage().contains("mapper [foo] of different type"));
     }
+
+    public void testMultiFieldsWithFieldAlias() throws Exception {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser();
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties")
+                .startObject("field")
+                    .field("type", "text")
+                    .startObject("fields")
+                        .startObject("alias")
+                            .field("type", "alias")
+                            .field("path", "other-field")
+                        .endObject()
+                    .endObject()
+                .endObject()
+                .startObject("other-field")
+                    .field("type", "keyword")
+                .endObject()
+            .endObject()
+        .endObject().endObject());
+        MapperParsingException e = expectThrows(MapperParsingException.class, () ->
+            mapperParser.parse("type", new CompressedXContent(mapping)));
+        assertEquals("Type [alias] cannot be used in multi field", e.getMessage());
+    }
 }

From 00a6ad0e9ee874bc211a2fc55824b0c14975c7a5 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Fri, 20 Jul 2018 09:23:32 +0200
Subject: [PATCH 111/260] Remove aliases resolution limitations when security
 is enabled (#31952)

Resolving wildcards in aliases expression is challenging as we may end
up with no aliases to replace the original expression with, but if we
replace with an empty array that means _all which is quite the opposite.
Now that we support and serialize the original requested aliases,
whenever aliases are replaced we will be able to know what was
initially requested. `MetaData#findAliases` can then be updated to not
return anything in case it gets empty aliases, but the original aliases
were not empty. That means that empty aliases are interpreted as _all
only if they were originally requested that way.

Relates to #31516
---
 .../migration/migrate_7_0/api.asciidoc        |  8 +++
 .../elasticsearch/action/AliasesRequest.java  |  5 ++
 .../indices/alias/IndicesAliasesRequest.java  | 16 ++++-
 .../alias/TransportIndicesAliasesAction.java  |  4 +-
 .../alias/get/TransportGetAliasesAction.java  |  2 +-
 .../indices/get/TransportGetIndexAction.java  |  5 +-
 .../cluster/metadata/MetaData.java            | 41 +++++++++++--
 .../cluster/metadata/MetaDataTests.java       | 59 ++++++++++++++++++
 x-pack/docs/en/security/limitations.asciidoc  |  2 -
 .../authz/IndicesAndAliasesResolver.java      | 20 ++-----
 .../security/authz/IndexAliasesTests.java     | 60 +++++++++++--------
 .../authz/IndicesAndAliasesResolverTests.java | 23 +++----
 .../build.gradle                              |  5 +-
 13 files changed, 181 insertions(+), 69 deletions(-)

diff --git a/docs/reference/migration/migrate_7_0/api.asciidoc b/docs/reference/migration/migrate_7_0/api.asciidoc
index f7b6f9b2e00..689b941ef6b 100644
--- a/docs/reference/migration/migrate_7_0/api.asciidoc
+++ b/docs/reference/migration/migrate_7_0/api.asciidoc
@@ -79,3 +79,11 @@ the only behavior in 8.0.0, this parameter is deprecated in 7.0.0 for removal in
 ==== The deprecated stored script contexts have now been removed
 When putting stored scripts, support for storing them with the deprecated `template` context or without a context is
 now removed. Scripts must be stored using the `script` context as mentioned in the documentation.
+
+==== Get Aliases API limitations when {security} is enabled removed
+
+The behavior and response codes of the get aliases API no longer vary
+depending on whether {security} is enabled. Previously a
+404 - NOT FOUND (IndexNotFoundException) could be returned in case the
+current user was not authorized for any alias. An empty response with
+status 200 - OK is now returned instead at all times.
diff --git a/server/src/main/java/org/elasticsearch/action/AliasesRequest.java b/server/src/main/java/org/elasticsearch/action/AliasesRequest.java
index bf7ceb28d50..419287f28eb 100644
--- a/server/src/main/java/org/elasticsearch/action/AliasesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/AliasesRequest.java
@@ -32,6 +32,11 @@ public interface AliasesRequest extends IndicesRequest.Replaceable {
      */
     String[] aliases();
 
+    /**
+     * Returns the aliases as they were originally requested, before any potential name resolution
+     */
+    String[] getOriginalAliases();
+
     /**
      * Replaces current aliases with the provided aliases.
      *
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
index 9249550871c..22e8554ed6a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -214,6 +214,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
         private final AliasActions.Type type;
         private String[] indices;
         private String[] aliases = Strings.EMPTY_ARRAY;
+        private String[] originalAliases = Strings.EMPTY_ARRAY;
         private String filter;
         private String routing;
         private String indexRouting;
@@ -238,6 +239,9 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
             if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
                 writeIndex = in.readOptionalBoolean();
             }
+            if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+                originalAliases = in.readStringArray();
+            }
         }
 
         @Override
@@ -252,6 +256,9 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
             if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
                 out.writeOptionalBoolean(writeIndex);
             }
+            if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+                out.writeStringArray(originalAliases);
+            }
         }
 
         /**
@@ -315,6 +322,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
                 }
             }
             this.aliases = aliases;
+            this.originalAliases = aliases;
             return this;
         }
 
@@ -329,6 +337,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
                 throw new IllegalArgumentException("[alias] can't be empty string");
             }
             this.aliases = new String[] {alias};
+            this.originalAliases = aliases;
             return this;
         }
 
@@ -432,6 +441,11 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
             this.aliases = aliases;
         }
 
+        @Override
+        public String[] getOriginalAliases() {
+            return originalAliases;
+        }
+
         @Override
         public boolean expandAliasesWildcards() {
             //remove operations support wildcards among aliases, add operations don't
@@ -579,7 +593,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
         }, AliasActions.PARSER, new ParseField("actions"));
     }
 
-    public static IndicesAliasesRequest fromXContent(XContentParser parser) throws IOException {
+    public static IndicesAliasesRequest fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
index c98e584c9c5..d137665c197 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
@@ -95,7 +95,7 @@ public class TransportIndicesAliasesAction extends TransportMasterNodeAction<Ind
         Set<String> aliases = new HashSet<>();
         for (AliasActions action : actions) {
             String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request.indicesOptions(), action.indices());
-            Collections.addAll(aliases, action.aliases());
+            Collections.addAll(aliases, action.getOriginalAliases());
             for (String index : concreteIndices) {
                 switch (action.actionType()) {
                 case ADD:
@@ -142,7 +142,7 @@ public class TransportIndicesAliasesAction extends TransportMasterNodeAction<Ind
         if (action.expandAliasesWildcards()) {
             //for DELETE we expand the aliases
             String[] indexAsArray = {concreteIndex};
-            ImmutableOpenMap<String, List<AliasMetaData>> aliasMetaData = metaData.findAliases(action.aliases(), indexAsArray);
+            ImmutableOpenMap<String, List<AliasMetaData>> aliasMetaData = metaData.findAliases(action, indexAsArray);
             List<String> finalAliases = new ArrayList<>();
             for (ObjectCursor<List<AliasMetaData>> curAliases : aliasMetaData.values()) {
                 for (AliasMetaData aliasMeta: curAliases.value) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
index 1bacd652ee7..2b71e85a537 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
@@ -63,7 +63,7 @@ public class TransportGetAliasesAction extends TransportMasterNodeReadAction<Get
     @Override
     protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener<GetAliasesResponse> listener) {
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
-        ImmutableOpenMap<String, List<AliasMetaData>> aliases = state.metaData().findAliases(request.aliases(), concreteIndices);
+        ImmutableOpenMap<String, List<AliasMetaData>> aliases = state.metaData().findAliases(request, concreteIndices);
         listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases)));
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
index 060c345454a..584ad0bc55a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
@@ -32,15 +32,14 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.common.settings.IndexScopedSettings;
 
 import java.io.IOException;
 import java.util.List;
@@ -110,7 +109,7 @@ public class TransportGetIndexAction extends TransportClusterInfoAction<GetIndex
                     break;
             case ALIASES:
                     if (!doneAliases) {
-                        aliasesResult = state.metaData().findAliases(Strings.EMPTY_ARRAY, concreteIndices);
+                        aliasesResult = state.metaData().findAllAliases(concreteIndices);
                         doneAliases = true;
                     }
                     break;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index c0243888683..43339599734 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -24,6 +24,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.action.AliasesRequest;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterState.FeatureAware;
 import org.elasticsearch.cluster.Diff;
@@ -248,21 +249,53 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
     }
 
     /**
-     * Finds the specific index aliases that match with the specified aliases directly or partially via wildcards and
-     * that point to the specified concrete indices or match partially with the indices via wildcards.
+     * Finds the specific index aliases that point to the specified concrete indices or match partially with the indices via wildcards.
      *
-     * @param aliases         The names of the index aliases to find
      * @param concreteIndices The concrete indexes the index aliases must point to order to be returned.
      * @return a map of index to a list of alias metadata, the list corresponding to a concrete index will be empty if no aliases are
      * present for that index
      */
-    public ImmutableOpenMap<String, List<AliasMetaData>> findAliases(final String[] aliases, String[] concreteIndices) {
+    public ImmutableOpenMap<String, List<AliasMetaData>> findAllAliases(String[] concreteIndices) {
+        return findAliases(Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, concreteIndices);
+    }
+
+    /**
+     * Finds the specific index aliases that match with the specified aliases directly or partially via wildcards and
+     * that point to the specified concrete indices or match partially with the indices via wildcards.
+     *
+     * @param aliasesRequest The request to find aliases for
+     * @param concreteIndices The concrete indexes the index aliases must point to order to be returned.
+     * @return a map of index to a list of alias metadata, the list corresponding to a concrete index will be empty if no aliases are
+     * present for that index
+     */
+    public ImmutableOpenMap<String, List<AliasMetaData>> findAliases(final AliasesRequest aliasesRequest, String[] concreteIndices) {
+        return findAliases(aliasesRequest.getOriginalAliases(), aliasesRequest.aliases(), concreteIndices);
+    }
+
+    /**
+     * Finds the specific index aliases that match with the specified aliases directly or partially via wildcards and
+     * that point to the specified concrete indices or match partially with the indices via wildcards.
+     *
+     * @param aliases The aliases to look for
+     * @param originalAliases The original aliases that the user originally requested
+     * @param concreteIndices The concrete indexes the index aliases must point to order to be returned.
+     * @return a map of index to a list of alias metadata, the list corresponding to a concrete index will be empty if no aliases are
+     * present for that index
+     */
+    private ImmutableOpenMap<String, List<AliasMetaData>> findAliases(String[] originalAliases, String[] aliases,
+                                                                      String[] concreteIndices) {
         assert aliases != null;
+        assert originalAliases != null;
         assert concreteIndices != null;
         if (concreteIndices.length == 0) {
             return ImmutableOpenMap.of();
         }
 
+        //if aliases were provided but they got replaced with empty aliases, return empty map
+        if (originalAliases.length > 0 && aliases.length == 0) {
+            return ImmutableOpenMap.of();
+        }
+
         boolean matchAllAliases = matchAllAliases(aliases);
         ImmutableOpenMap.Builder<String, List<AliasMetaData>> mapBuilder = ImmutableOpenMap.builder();
         for (String index : concreteIndices) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 38e3fcc6ea7..0b9055cb758 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -41,6 +42,7 @@ import org.elasticsearch.test.ESTestCase;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -50,6 +52,63 @@ import static org.hamcrest.Matchers.startsWith;
 
 public class MetaDataTests extends ESTestCase {
 
+    public void testFindAliases() {
+        MetaData metaData = MetaData.builder().put(IndexMetaData.builder("index")
+            .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putAlias(AliasMetaData.builder("alias1").build())
+            .putAlias(AliasMetaData.builder("alias2").build())).build();
+
+        {
+            ImmutableOpenMap<String, List<AliasMetaData>> aliases = metaData.findAliases(new GetAliasesRequest(), Strings.EMPTY_ARRAY);
+            assertThat(aliases.size(), equalTo(0));
+        }
+        {
+            ImmutableOpenMap<String, List<AliasMetaData>> aliases = metaData.findAliases(new GetAliasesRequest(), new String[]{"index"});
+            assertThat(aliases.size(), equalTo(1));
+            List<AliasMetaData> aliasMetaDataList = aliases.get("index");
+            assertThat(aliasMetaDataList.size(), equalTo(2));
+            assertThat(aliasMetaDataList.get(0).alias(), equalTo("alias1"));
+            assertThat(aliasMetaDataList.get(1).alias(), equalTo("alias2"));
+        }
+        {
+            GetAliasesRequest getAliasesRequest = new GetAliasesRequest("alias1");
+            getAliasesRequest.replaceAliases(Strings.EMPTY_ARRAY);
+            ImmutableOpenMap<String, List<AliasMetaData>> aliases = metaData.findAliases(getAliasesRequest, new String[]{"index"});
+            assertThat(aliases.size(), equalTo(0));
+        }
+        {
+            ImmutableOpenMap<String, List<AliasMetaData>> aliases =
+                metaData.findAliases(new GetAliasesRequest("alias*"), new String[]{"index"});
+            assertThat(aliases.size(), equalTo(1));
+            List<AliasMetaData> aliasMetaDataList = aliases.get("index");
+            assertThat(aliasMetaDataList.size(), equalTo(2));
+            assertThat(aliasMetaDataList.get(0).alias(), equalTo("alias1"));
+            assertThat(aliasMetaDataList.get(1).alias(), equalTo("alias2"));
+        }
+        {
+            ImmutableOpenMap<String, List<AliasMetaData>> aliases =
+                metaData.findAliases(new GetAliasesRequest("alias1"), new String[]{"index"});
+            assertThat(aliases.size(), equalTo(1));
+            List<AliasMetaData> aliasMetaDataList = aliases.get("index");
+            assertThat(aliasMetaDataList.size(), equalTo(1));
+            assertThat(aliasMetaDataList.get(0).alias(), equalTo("alias1"));
+        }
+        {
+            ImmutableOpenMap<String, List<AliasMetaData>> aliases = metaData.findAllAliases(new String[]{"index"});
+            assertThat(aliases.size(), equalTo(1));
+            List<AliasMetaData> aliasMetaDataList = aliases.get("index");
+            assertThat(aliasMetaDataList.size(), equalTo(2));
+            assertThat(aliasMetaDataList.get(0).alias(), equalTo("alias1"));
+            assertThat(aliasMetaDataList.get(1).alias(), equalTo("alias2"));
+        }
+        {
+            ImmutableOpenMap<String, List<AliasMetaData>> aliases = metaData.findAllAliases(Strings.EMPTY_ARRAY);
+            assertThat(aliases.size(), equalTo(0));
+        }
+    }
+
     public void testIndexAndAliasWithSameName() {
         IndexMetaData.Builder builder = IndexMetaData.builder("index")
                 .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
diff --git a/x-pack/docs/en/security/limitations.asciidoc b/x-pack/docs/en/security/limitations.asciidoc
index c127ee3d796..fb8b826d5dd 100644
--- a/x-pack/docs/en/security/limitations.asciidoc
+++ b/x-pack/docs/en/security/limitations.asciidoc
@@ -19,8 +19,6 @@ with {security} enabled.
 Elasticsearch clusters with {security} enabled apply the `/_all` wildcard, and
 all other wildcards, to the indices that the current user has privileges for, not
 the set of all indices on the cluster.
-While creating or retrieving aliases by providing wildcard expressions for alias names, if there are no existing authorized aliases
-that match the wildcard expression provided an IndexNotFoundException is returned.
 
 [float]
 === Multi Document APIs
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
index 77170f7a1cb..2247cbe02a8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -20,7 +20,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -200,6 +199,8 @@ class IndicesAndAliasesResolver {
             if (aliasesRequest.expandAliasesWildcards()) {
                 List<String> aliases = replaceWildcardsWithAuthorizedAliases(aliasesRequest.aliases(),
                         loadAuthorizedAliases(authorizedIndices.get(), metaData));
+                //it may be that we replace aliases with an empty array, in case there are no authorized aliases for the action.
+                //MetaData#findAliases will return nothing when some alias was originally requested, which was replaced with empty.
                 aliasesRequest.replaceAliases(aliases.toArray(new String[aliases.size()]));
             }
             if (indicesReplacedWithNoIndices) {
@@ -240,8 +241,7 @@ class IndicesAndAliasesResolver {
         } else {
             // the user is not authorized to put mappings for this index, but could have been
             // authorized for a write using an alias that triggered a dynamic mapping update
-            ImmutableOpenMap<String, List<AliasMetaData>> foundAliases =
-                metaData.findAliases(Strings.EMPTY_ARRAY, new String[] { concreteIndexName });
+            ImmutableOpenMap<String, List<AliasMetaData>> foundAliases = metaData.findAllAliases(new String[] { concreteIndexName });
             List<AliasMetaData> aliasMetaData = foundAliases.get(concreteIndexName);
             if (aliasMetaData != null) {
                 Optional<String> foundAlias = aliasMetaData.stream()
@@ -279,14 +279,12 @@ class IndicesAndAliasesResolver {
         List<String> finalAliases = new ArrayList<>();
 
         //IndicesAliasesRequest doesn't support empty aliases (validation fails) but GetAliasesRequest does (in which case empty means _all)
-        boolean matchAllAliases = aliases.length == 0;
-        if (matchAllAliases) {
+        if (aliases.length == 0) {
             finalAliases.addAll(authorizedAliases);
         }
 
         for (String aliasPattern : aliases) {
             if (aliasPattern.equals(MetaData.ALL)) {
-                matchAllAliases = true;
                 finalAliases.addAll(authorizedAliases);
             } else if (Regex.isSimpleMatchPattern(aliasPattern)) {
                 for (String authorizedAlias : authorizedAliases) {
@@ -298,16 +296,6 @@ class IndicesAndAliasesResolver {
                 finalAliases.add(aliasPattern);
             }
         }
-
-        //Throw exception if the wildcards expansion to authorized aliases resulted in no indices.
-        //We always need to replace wildcards for security reasons, to make sure that the operation is executed on the aliases that we
-        //authorized it to execute on. Empty set gets converted to _all by es core though, and unlike with indices, here we don't have
-        //a special expression to replace empty set with, which gives us the guarantee that nothing will be returned.
-        //This is because existing aliases can contain all kinds of special characters, they are only validated since 5.1.
-        if (finalAliases.isEmpty()) {
-            String indexName = matchAllAliases ? MetaData.ALL : Arrays.toString(aliases);
-            throw new IndexNotFoundException(indexName);
-        }
         return finalAliases;
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
index c6cb8bb662c..036f1667e14 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
@@ -15,6 +15,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.junit.Before;
 
@@ -235,15 +236,19 @@ public class IndexAliasesTests extends SecurityIntegTestCase {
         //ok: user has manage_aliases on test_*
         assertAcked(client().filterWithHeader(headers).admin().indices().prepareAliases().removeAlias("test_1", "test_alias_*").get());
 
-        //fails: all aliases have been deleted, no existing aliases match test_alias_*
-        IndexNotFoundException indexNotFoundException = expectThrows(IndexNotFoundException.class,
+        {
+            //fails: all aliases have been deleted, no existing aliases match test_alias_*
+            AliasesNotFoundException exception = expectThrows(AliasesNotFoundException.class,
                 client().filterWithHeader(headers).admin().indices().prepareAliases().removeAlias("test_1", "test_alias_*")::get);
-        assertThat(indexNotFoundException.toString(), containsString("[test_alias_*]"));
+            assertThat(exception.getMessage(), equalTo("aliases [test_alias_*] missing"));
+        }
 
-        //fails: all aliases have been deleted, no existing aliases match _all
-        indexNotFoundException = expectThrows(IndexNotFoundException.class,
+        {
+            //fails: all aliases have been deleted, no existing aliases match _all
+            AliasesNotFoundException exception = expectThrows(AliasesNotFoundException.class,
                 client().filterWithHeader(headers).admin().indices().prepareAliases().removeAlias("test_1", "_all")::get);
-        assertThat(indexNotFoundException.toString(), containsString("[_all]"));
+            assertThat(exception.getMessage(), equalTo("aliases [_all] missing"));
+        }
 
         //fails: user doesn't have manage_aliases on alias_1
         assertThrowsAuthorizationException(client().filterWithHeader(headers).admin().indices().prepareAliases()
@@ -383,24 +388,27 @@ public class IndexAliasesTests extends SecurityIntegTestCase {
         getAliasesResponse = client.admin().indices().prepareGetAliases().setAliases("test_alias").get();
         assertEquals(0, getAliasesResponse.getAliases().size());
 
-        //fails: no existing aliases to replace wildcards
-        IndexNotFoundException indexNotFoundException = expectThrows(IndexNotFoundException.class,
-                client.admin().indices().prepareGetAliases().setIndices("test_1").setAliases("test_*")::get);
-        assertThat(indexNotFoundException.toString(), containsString("[test_*]"));
-
-        //fails: no existing aliases to replace _all
-        indexNotFoundException = expectThrows(IndexNotFoundException.class,
-                client.admin().indices().prepareGetAliases().setIndices("test_1").setAliases("_all")::get);
-        assertThat(indexNotFoundException.toString(), containsString("[_all]"));
-
-        //fails: no existing aliases to replace empty aliases
-        indexNotFoundException = expectThrows(IndexNotFoundException.class,
-                client.admin().indices().prepareGetAliases().setIndices("test_1")::get);
-        assertThat(indexNotFoundException.toString(), containsString("[_all]"));
-
-        //fails: no existing aliases to replace empty aliases
-        indexNotFoundException = expectThrows(IndexNotFoundException.class, client.admin().indices().prepareGetAliases()::get);
-        assertThat(indexNotFoundException.toString(), containsString("[_all]"));
+        {
+            //fails: no existing aliases to replace wildcards
+            assertThrowsAuthorizationException(
+                client.admin().indices().prepareGetAliases().setIndices("test_1").setAliases("test_*")::get,
+                GetAliasesAction.NAME, "create_test_aliases_alias");
+        }
+        {
+            //fails: no existing aliases to replace _all
+            assertThrowsAuthorizationException(client.admin().indices().prepareGetAliases().setIndices("test_1").setAliases("_all")::get,
+                GetAliasesAction.NAME, "create_test_aliases_alias");
+        }
+        {
+            //fails: no existing aliases to replace empty aliases
+            assertThrowsAuthorizationException(client.admin().indices().prepareGetAliases().setIndices("test_1")::get,
+                GetAliasesAction.NAME, "create_test_aliases_alias");
+        }
+        {
+            //fails: no existing aliases to replace empty aliases
+            GetAliasesResponse response = client.admin().indices().prepareGetAliases().get();
+            assertThat(response.getAliases().size(), equalTo(0));
+        }
     }
 
     public void testCreateIndexThenAliasesCreateAndAliasesPermission3() {
@@ -447,9 +455,9 @@ public class IndexAliasesTests extends SecurityIntegTestCase {
         assertAcked(client.admin().indices().prepareAliases().removeAlias("test_*", "_all"));
 
         //fails: all aliases have been deleted, _all can't be resolved to any existing authorized aliases
-        IndexNotFoundException indexNotFoundException = expectThrows(IndexNotFoundException.class,
+        AliasesNotFoundException exception = expectThrows(AliasesNotFoundException.class,
                 client.admin().indices().prepareAliases().removeAlias("test_1", "_all")::get);
-        assertThat(indexNotFoundException.toString(), containsString("[_all]"));
+        assertThat(exception.getMessage(), equalTo("aliases [_all] missing"));
     }
 
     public void testGetAliasesCreateAndAliasesPermission3() {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
index d7c974bdc6e..bd5acdec818 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
@@ -80,6 +80,7 @@ import java.util.Set;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
@@ -781,10 +782,11 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     public void testResolveAliasesWildcardsIndicesAliasesRequestDeleteActionsNoAuthorizedIndices() {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo*").alias("foo*"));
-        //no authorized aliases match bar*, hence this action fails and makes the whole request fail
+        //no authorized aliases match bar*, hence aliases are replaced with empty string for that action
         request.addAliasAction(AliasActions.remove().index("*bar").alias("bar*"));
-        expectThrows(IndexNotFoundException.class, () -> resolveIndices(
-                request, buildAuthorizedIndices(user, IndicesAliasesAction.NAME)));
+        resolveIndices(request, buildAuthorizedIndices(user, IndicesAliasesAction.NAME));
+        assertThat(request.getAliasActions().get(0).aliases().length, equalTo(1));
+        assertThat(request.getAliasActions().get(1).aliases().length, equalTo(0));
     }
 
     public void testResolveWildcardsIndicesAliasesRequestAddAndDeleteActions() {
@@ -1086,12 +1088,11 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
 
     public void testResolveAliasesWildcardsGetAliasesRequestNoAuthorizedIndices() {
         GetAliasesRequest request = new GetAliasesRequest();
-        //no authorized aliases match bar*, hence the request fails
+        //no authorized aliases match bar*, hence aliases are replaced with empty array
         request.aliases("bar*");
         request.indices("*bar");
-        IndexNotFoundException e = expectThrows(IndexNotFoundException.class,
-                () -> resolveIndices(request, buildAuthorizedIndices(user, GetAliasesAction.NAME)));
-        assertEquals("no such index", e.getMessage());
+        resolveIndices(request, buildAuthorizedIndices(user, GetAliasesAction.NAME));
+        assertThat(request.aliases().length, equalTo(0));
     }
 
     public void testResolveAliasesAllGetAliasesRequestNoAuthorizedIndices() {
@@ -1100,10 +1101,10 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
             request.aliases("_all");
         }
         request.indices("non_existing");
-        //current user is not authorized for any index, foo* resolves to no indices, the request fails
-        IndexNotFoundException e = expectThrows(IndexNotFoundException.class,
-                () -> resolveIndices(request, buildAuthorizedIndices(userNoIndices, GetAliasesAction.NAME)));
-        assertEquals("no such index", e.getMessage());
+        //current user is not authorized for any index, foo* resolves to no indices, aliases are replaced with empty array
+        ResolvedIndices resolvedIndices = resolveIndices(request, buildAuthorizedIndices(userNoIndices, GetAliasesAction.NAME));
+        assertThat(resolvedIndices.getLocal(), contains("non_existing"));
+        assertThat(request.aliases().length, equalTo(0));
     }
 
     /**
diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle
index 7f2706a773a..35b08de0125 100644
--- a/x-pack/qa/core-rest-tests-with-security/build.gradle
+++ b/x-pack/qa/core-rest-tests-with-security/build.gradle
@@ -12,10 +12,9 @@ integTest {
 
 integTestRunner {
   systemProperty 'tests.rest.blacklist',
-    ['cat.aliases/10_basic/Empty cluster',
+    [
      'index/10_with_id/Index with ID',
-     'indices.get_alias/10_basic/Get alias against closed indices',
-     'indices.get_alias/20_empty/Check empty aliases when getting all aliases via /_alias',
+     'indices.get_alias/10_basic/Get alias against closed indices'
     ].join(',')
 
   systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')

From c5cde96691df84353f1f796db40c9df0efc6825e Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Fri, 20 Jul 2018 10:18:38 +0200
Subject: [PATCH 112/260] Dependencies: Upgrade to joda time 2.10 (#32160)

Changelog: http://www.joda.org/joda-time/changes-report.html
---
 server/build.gradle                                            | 2 +-
 server/licenses/joda-time-2.10.jar.sha1                        | 1 +
 server/licenses/joda-time-2.9.9.jar.sha1                       | 1 -
 x-pack/plugin/sql/sql-action/build.gradle                      | 2 +-
 x-pack/plugin/sql/sql-action/licenses/joda-time-2.10.jar.sha1  | 1 +
 x-pack/plugin/sql/sql-action/licenses/joda-time-2.9.9.jar.sha1 | 1 -
 x-pack/plugin/sql/sql-proto/build.gradle                       | 2 +-
 x-pack/plugin/sql/sql-proto/licenses/joda-time-2.10.jar.sha1   | 1 +
 x-pack/plugin/sql/sql-proto/licenses/joda-time-2.9.9.jar.sha1  | 1 -
 9 files changed, 6 insertions(+), 6 deletions(-)
 create mode 100644 server/licenses/joda-time-2.10.jar.sha1
 delete mode 100644 server/licenses/joda-time-2.9.9.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/joda-time-2.10.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/joda-time-2.9.9.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-proto/licenses/joda-time-2.10.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-proto/licenses/joda-time-2.9.9.jar.sha1

diff --git a/server/build.gradle b/server/build.gradle
index c71cc4c7dbd..deb38398979 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -106,7 +106,7 @@ dependencies {
   compile 'com.carrotsearch:hppc:0.7.1'
 
   // time handling, remove with java 8 time
-  compile 'joda-time:joda-time:2.9.9'
+  compile 'joda-time:joda-time:2.10'
 
   // percentiles aggregation
   compile 'com.tdunning:t-digest:3.2'
diff --git a/server/licenses/joda-time-2.10.jar.sha1 b/server/licenses/joda-time-2.10.jar.sha1
new file mode 100644
index 00000000000..a597eabc654
--- /dev/null
+++ b/server/licenses/joda-time-2.10.jar.sha1
@@ -0,0 +1 @@
+f66c8125d1057ffce6c4e29e624cac863e110e2b
\ No newline at end of file
diff --git a/server/licenses/joda-time-2.9.9.jar.sha1 b/server/licenses/joda-time-2.9.9.jar.sha1
deleted file mode 100644
index 4009932ea3b..00000000000
--- a/server/licenses/joda-time-2.9.9.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7b520c458572890807d143670c9b24f4de90897
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle
index d8805d2e3db..f6b5177d508 100644
--- a/x-pack/plugin/sql/sql-action/build.gradle
+++ b/x-pack/plugin/sql/sql-action/build.gradle
@@ -24,7 +24,7 @@ dependencies {
     }
     compile xpackProject('plugin:sql:sql-proto')
     compile "org.apache.lucene:lucene-core:${versions.lucene}"
-    compile 'joda-time:joda-time:2.9.9'
+    compile 'joda-time:joda-time:2.10'
     runtime "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
     runtime "org.apache.logging.log4j:log4j-api:${versions.log4j}"
     runtime "org.apache.logging.log4j:log4j-core:${versions.log4j}"
diff --git a/x-pack/plugin/sql/sql-action/licenses/joda-time-2.10.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/joda-time-2.10.jar.sha1
new file mode 100644
index 00000000000..a597eabc654
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/joda-time-2.10.jar.sha1
@@ -0,0 +1 @@
+f66c8125d1057ffce6c4e29e624cac863e110e2b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/joda-time-2.9.9.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/joda-time-2.9.9.jar.sha1
deleted file mode 100644
index 4009932ea3b..00000000000
--- a/x-pack/plugin/sql/sql-action/licenses/joda-time-2.9.9.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7b520c458572890807d143670c9b24f4de90897
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-proto/build.gradle b/x-pack/plugin/sql/sql-proto/build.gradle
index 5a1439f4360..7f26176e3c7 100644
--- a/x-pack/plugin/sql/sql-proto/build.gradle
+++ b/x-pack/plugin/sql/sql-proto/build.gradle
@@ -17,7 +17,7 @@ dependencies {
     compile (project(':libs:x-content')) {
         transitive = false
     }
-    compile 'joda-time:joda-time:2.9.9'
+    compile 'joda-time:joda-time:2.10'
     runtime "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
 
     testCompile "org.elasticsearch.test:framework:${version}"
diff --git a/x-pack/plugin/sql/sql-proto/licenses/joda-time-2.10.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/joda-time-2.10.jar.sha1
new file mode 100644
index 00000000000..a597eabc654
--- /dev/null
+++ b/x-pack/plugin/sql/sql-proto/licenses/joda-time-2.10.jar.sha1
@@ -0,0 +1 @@
+f66c8125d1057ffce6c4e29e624cac863e110e2b
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-proto/licenses/joda-time-2.9.9.jar.sha1 b/x-pack/plugin/sql/sql-proto/licenses/joda-time-2.9.9.jar.sha1
deleted file mode 100644
index 4009932ea3b..00000000000
--- a/x-pack/plugin/sql/sql-proto/licenses/joda-time-2.9.9.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f7b520c458572890807d143670c9b24f4de90897
\ No newline at end of file

From 5cbd9ad177e4284a2fef799d01920bd09648de70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Fri, 20 Jul 2018 11:43:46 +0200
Subject: [PATCH 113/260] Rename ranking evaluation response section (#32166)

Currently the ranking evaluation response contains a 'unknown_docs' section
for each search use case in the evaluation set. It contains document ids for
results in the search hits that currently don't have a quality rating.
This change renames it to `unrated_docs`, which better reflects its purpose.
---
 .../test/java/org/elasticsearch/client/RankEvalIT.java |  6 +++---
 docs/reference/search/rank-eval.asciidoc               |  4 ++--
 .../elasticsearch/index/rankeval/EvalQueryQuality.java |  6 +++---
 .../elasticsearch/index/rankeval/EvaluationMetric.java |  5 ++---
 .../index/rankeval/DiscountedCumulativeGainTests.java  | 10 +++++-----
 .../index/rankeval/EvalQueryQualityTests.java          |  6 ------
 .../index/rankeval/RankEvalRequestIT.java              |  6 +++---
 .../index/rankeval/RankEvalResponseTests.java          |  2 +-
 .../rest-api-spec/test/rank_eval/10_basic.yml          |  8 ++++----
 .../resources/rest-api-spec/test/rank_eval/20_dcg.yml  |  8 ++++----
 .../rest-api-spec/test/rank_eval/30_failures.yml       |  2 +-
 .../test/rank_eval/40_rank_eval_templated.yml          |  2 +-
 12 files changed, 29 insertions(+), 36 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
index d61fccb9371..2890257b236 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
@@ -40,7 +40,7 @@ import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
+import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnratedDocuments;
 
 public class RankEvalIT extends ESRestHighLevelClientTestCase {
 
@@ -84,7 +84,7 @@ public class RankEvalIT extends ESRestHighLevelClientTestCase {
         Map<String, EvalQueryQuality> partialResults = response.getPartialResults();
         assertEquals(2, partialResults.size());
         EvalQueryQuality amsterdamQueryQuality = partialResults.get("amsterdam_query");
-        assertEquals(2, filterUnknownDocuments(amsterdamQueryQuality.getHitsAndRatings()).size());
+        assertEquals(2, filterUnratedDocuments(amsterdamQueryQuality.getHitsAndRatings()).size());
         List<RatedSearchHit> hitsAndRatings = amsterdamQueryQuality.getHitsAndRatings();
         assertEquals(7, hitsAndRatings.size());
         for (RatedSearchHit hit : hitsAndRatings) {
@@ -96,7 +96,7 @@ public class RankEvalIT extends ESRestHighLevelClientTestCase {
             }
         }
         EvalQueryQuality berlinQueryQuality = partialResults.get("berlin_query");
-        assertEquals(6, filterUnknownDocuments(berlinQueryQuality.getHitsAndRatings()).size());
+        assertEquals(6, filterUnratedDocuments(berlinQueryQuality.getHitsAndRatings()).size());
         hitsAndRatings = berlinQueryQuality.getHitsAndRatings();
         assertEquals(7, hitsAndRatings.size());
         for (RatedSearchHit hit : hitsAndRatings) {
diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
index 571a4886991..cf13b9f7b06 100644
--- a/docs/reference/search/rank-eval.asciidoc
+++ b/docs/reference/search/rank-eval.asciidoc
@@ -274,7 +274,7 @@ that shows potential errors of individual queries. The response has the followin
         "details": {  
             "my_query_id1": { <2>
                 "quality_level": 0.6, <3>
-                "unknown_docs": [ <4>
+                "unrated_docs": [ <4>
                     {
                         "_index": "my_index",
                         "_id": "1960795"
@@ -309,7 +309,7 @@ that shows potential errors of individual queries. The response has the followin
 <1> the overall evaluation quality calculated by the defined metric
 <2> the `details` section contains one entry for every query in the original `requests` section, keyed by the search request id
 <3> the `quality_level` in the `details` section shows the contribution of this query to the global quality score
-<4> the `unknown_docs` section contains an `_index` and `_id` entry for each document in the search result for this
+<4> the `unrated_docs` section contains an `_index` and `_id` entry for each document in the search result for this
 query that didn't have a ratings value. This can be used to ask the user to supply ratings for these documents
 <5> the `hits` section shows a grouping of the search results with their supplied rating
 <6> the `metric_details` give additional information about the calculated quality metric (e.g. how many of the retrieved
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
index 2ad3e589bd8..91ba1ce6169 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
@@ -102,8 +102,8 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(queryId);
         builder.field(QUALITY_LEVEL_FIELD.getPreferredName(), this.evaluationResult);
-        builder.startArray(UNKNOWN_DOCS_FIELD.getPreferredName());
-        for (DocumentKey key : EvaluationMetric.filterUnknownDocuments(ratedHits)) {
+        builder.startArray(UNRATED_DOCS_FIELD.getPreferredName());
+        for (DocumentKey key : EvaluationMetric.filterUnratedDocuments(ratedHits)) {
             builder.startObject();
             builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), key.getIndex());
             builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), key.getDocId());
@@ -123,7 +123,7 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
     }
 
     private static final ParseField QUALITY_LEVEL_FIELD = new ParseField("quality_level");
-    private static final ParseField UNKNOWN_DOCS_FIELD = new ParseField("unknown_docs");
+    private static final ParseField UNRATED_DOCS_FIELD = new ParseField("unrated_docs");
     private static final ParseField HITS_FIELD = new ParseField("hits");
     private static final ParseField METRIC_DETAILS_FIELD = new ParseField("metric_details");
     private static final ObjectParser<ParsedEvalQueryQuality, Void> PARSER = new ObjectParser<>("eval_query_quality",
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
index c67511e051f..37898fd9516 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
@@ -76,10 +76,9 @@ public interface EvaluationMetric extends ToXContentObject, NamedWriteable {
     /**
      * filter @link {@link RatedSearchHit} that don't have a rating
      */
-    static List<DocumentKey> filterUnknownDocuments(List<RatedSearchHit> ratedHits) {
-        List<DocumentKey> unknownDocs = ratedHits.stream().filter(hit -> hit.getRating().isPresent() == false)
+    static List<DocumentKey> filterUnratedDocuments(List<RatedSearchHit> ratedHits) {
+        return ratedHits.stream().filter(hit -> hit.getRating().isPresent() == false)
                 .map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), hit.getSearchHit().getId())).collect(Collectors.toList());
-        return unknownDocs;
     }
 
     /**
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
index 56b0c692c41..e768c297333 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
@@ -40,7 +40,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
+import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnratedDocuments;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.hamcrest.CoreMatchers.containsString;
@@ -128,7 +128,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         EvalQueryQuality result = dcg.evaluate("id", hits, rated);
         assertEquals(12.779642067948913, result.getQualityLevel(), DELTA);
-        assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size());
+        assertEquals(2, filterUnratedDocuments(result.getHitsAndRatings()).size());
 
         /**
          * Check with normalization: to get the maximal possible dcg, sort documents by
@@ -185,7 +185,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs);
         assertEquals(12.392789260714371, result.getQualityLevel(), DELTA);
-        assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size());
+        assertEquals(1, filterUnratedDocuments(result.getHitsAndRatings()).size());
 
         /**
          * Check with normalization: to get the maximal possible dcg, sort documents by
@@ -224,13 +224,13 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs);
         assertEquals(0.0d, result.getQualityLevel(), DELTA);
-        assertEquals(0, filterUnknownDocuments(result.getHitsAndRatings()).size());
+        assertEquals(0, filterUnratedDocuments(result.getHitsAndRatings()).size());
 
         // also check normalized
         dcg = new DiscountedCumulativeGain(true, null, 10);
         result = dcg.evaluate("id", hits, ratedDocs);
         assertEquals(0.0d, result.getQualityLevel(), DELTA);
-        assertEquals(0, filterUnknownDocuments(result.getHitsAndRatings()).size());
+        assertEquals(0, filterUnratedDocuments(result.getHitsAndRatings()).size());
     }
 
     public void testParseFromXContent() throws IOException {
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
index e9fae6b5c63..c9251bb8090 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
 
@@ -52,11 +51,6 @@ public class EvalQueryQualityTests extends ESTestCase {
     }
 
     public static EvalQueryQuality randomEvalQueryQuality() {
-        List<DocumentKey> unknownDocs = new ArrayList<>();
-        int numberOfUnknownDocs = randomInt(5);
-        for (int i = 0; i < numberOfUnknownDocs; i++) {
-            unknownDocs.add(new DocumentKey(randomAlphaOfLength(10), randomAlphaOfLength(10)));
-        }
         int numberOfSearchHits = randomInt(5);
         List<RatedSearchHit> ratedHits = new ArrayList<>();
         for (int i = 0; i < numberOfSearchHits; i++) {
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
index b55c57bae2b..28200e7d5a0 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
@@ -40,7 +40,7 @@ import java.util.List;
 import java.util.Map.Entry;
 import java.util.Set;
 
-import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
+import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnratedDocuments;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -120,7 +120,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         for (Entry<String, EvalQueryQuality> entry : entrySet) {
             EvalQueryQuality quality = entry.getValue();
             if (entry.getKey() == "amsterdam_query") {
-                assertEquals(2, filterUnknownDocuments(quality.getHitsAndRatings()).size());
+                assertEquals(2, filterUnratedDocuments(quality.getHitsAndRatings()).size());
                 List<RatedSearchHit> hitsAndRatings = quality.getHitsAndRatings();
                 assertEquals(6, hitsAndRatings.size());
                 for (RatedSearchHit hit : hitsAndRatings) {
@@ -133,7 +133,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
                 }
             }
             if (entry.getKey() == "berlin_query") {
-                assertEquals(5, filterUnknownDocuments(quality.getHitsAndRatings()).size());
+                assertEquals(5, filterUnratedDocuments(quality.getHitsAndRatings()).size());
                 List<RatedSearchHit> hitsAndRatings = quality.getHitsAndRatings();
                 assertEquals(6, hitsAndRatings.size());
                 for (RatedSearchHit hit : hitsAndRatings) {
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
index e4fe4848237..1e94e869d25 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
@@ -158,7 +158,7 @@ public class RankEvalResponseTests extends ESTestCase {
                 "    \"details\": {" +
                 "        \"coffee_query\": {" +
                 "            \"quality_level\": 0.1," +
-                "            \"unknown_docs\": [{\"_index\":\"index\",\"_id\":\"456\"}]," +
+                "            \"unrated_docs\": [{\"_index\":\"index\",\"_id\":\"456\"}]," +
                 "            \"hits\":[{\"hit\":{\"_index\":\"index\",\"_type\":\"\",\"_id\":\"123\",\"_score\":1.0}," +
                 "                       \"rating\":5}," +
                 "                      {\"hit\":{\"_index\":\"index\",\"_type\":\"\",\"_id\":\"456\",\"_score\":1.0}," +
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
index 3900b1f32ba..62c246fb320 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
@@ -73,7 +73,7 @@ setup:
 
   - match: { quality_level: 1}
   - match: { details.amsterdam_query.quality_level: 1.0}
-  - match: { details.amsterdam_query.unknown_docs:  [ {"_index": "foo", "_id": "doc4"}]}
+  - match: { details.amsterdam_query.unrated_docs:  [ {"_index": "foo", "_id": "doc4"}]}
   - match: { details.amsterdam_query.metric_details.precision: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}}
   
   - length: { details.amsterdam_query.hits: 3}
@@ -85,7 +85,7 @@ setup:
   - is_false: details.amsterdam_query.hits.2.rating
   
   - match: { details.berlin_query.quality_level: 1.0}
-  - match: { details.berlin_query.unknown_docs:  [ {"_index": "foo", "_id": "doc4"}]}
+  - match: { details.berlin_query.unrated_docs:  [ {"_index": "foo", "_id": "doc4"}]}
   - match: { details.berlin_query.metric_details.precision: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}}
   - length: { details.berlin_query.hits: 2}
   - match: { details.berlin_query.hits.0.hit._id: "doc1" }
@@ -155,9 +155,9 @@ setup:
   - gt: {details.amsterdam_query.quality_level: 0.333}
   - lt: {details.amsterdam_query.quality_level: 0.334}
   - match: {details.amsterdam_query.metric_details.mean_reciprocal_rank: {"first_relevant": 3}}
-  - match: {details.amsterdam_query.unknown_docs:  [ {"_index": "foo", "_id": "doc2"},
+  - match: {details.amsterdam_query.unrated_docs:  [ {"_index": "foo", "_id": "doc2"},
                                                                {"_index": "foo", "_id": "doc3"} ]}
   - match: {details.berlin_query.quality_level: 0.5}
   - match: {details.berlin_query.metric_details.mean_reciprocal_rank: {"first_relevant": 2}}
-  - match: {details.berlin_query.unknown_docs:  [ {"_index": "foo", "_id": "doc1"}]}
+  - match: {details.berlin_query.unrated_docs:  [ {"_index": "foo", "_id": "doc1"}]}
 
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
index fc5e6576ad4..baf10f1542c 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
@@ -73,7 +73,7 @@
   - lt: {quality_level: 13.848264 }
   - gt: {details.dcg_query.quality_level: 13.848263}
   - lt: {details.dcg_query.quality_level: 13.848264}
-  - match: {details.dcg_query.unknown_docs: [ ]}
+  - match: {details.dcg_query.unrated_docs: [ ]}
 
 # reverse the order in which the results are returned (less relevant docs first)
 
@@ -100,7 +100,7 @@
   - lt: {quality_level: 10.299675}
   - gt: {details.dcg_query_reverse.quality_level: 10.299674}
   - lt: {details.dcg_query_reverse.quality_level: 10.299675}
-  - match: {details.dcg_query_reverse.unknown_docs: [ ]}
+  - match: {details.dcg_query_reverse.unrated_docs: [ ]}
 
 # if we mix both, we should get the average
   
@@ -138,7 +138,7 @@
   - lt: {quality_level: 12.073970}
   - gt: {details.dcg_query.quality_level: 13.848263}
   - lt: {details.dcg_query.quality_level: 13.848264}
-  - match: {details.dcg_query.unknown_docs: [ ]}
+  - match: {details.dcg_query.unrated_docs: [ ]}
   - gt: {details.dcg_query_reverse.quality_level: 10.299674}
   - lt: {details.dcg_query_reverse.quality_level: 10.299675}
-  - match: {details.dcg_query_reverse.unknown_docs: [ ]}
+  - match: {details.dcg_query_reverse.unrated_docs: [ ]}
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
index 4008f677185..d6119ad3a9e 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
@@ -36,7 +36,7 @@
 
   - match: { quality_level: 1}
   - match: { details.amsterdam_query.quality_level: 1.0}
-  - match: { details.amsterdam_query.unknown_docs:  [ ]}
+  - match: { details.amsterdam_query.unrated_docs:  [ ]}
   - match: { details.amsterdam_query.metric_details.precision: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}}
 
   - is_true: failures.invalid_query
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml
index f0c564d3639..5e0082d213c 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml
@@ -85,7 +85,7 @@ setup:
         }
 
   - match: {quality_level: 0.9}
-  - match: {details.amsterdam_query.unknown_docs.0._id:  "6"}
+  - match: {details.amsterdam_query.unrated_docs.0._id:  "6"}
 
 ---
 "Test illegal request parts":

From 6afb661ca5393853df38d9a104568e1ea8b34d0a Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@gmail.com>
Date: Fri, 20 Jul 2018 12:55:20 +0200
Subject: [PATCH 114/260] Remove indices stats timeout from monitoring docs

With this commit we remove the documentation for the setting
`xpack.monitoring.collection.indices.stats.timeout` which has already
been removed in code.

Closes #32133
Relates #32229
---
 docs/reference/settings/monitoring-settings.asciidoc | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/reference/settings/monitoring-settings.asciidoc b/docs/reference/settings/monitoring-settings.asciidoc
index 5c812b6f53c..2759944e615 100644
--- a/docs/reference/settings/monitoring-settings.asciidoc
+++ b/docs/reference/settings/monitoring-settings.asciidoc
@@ -85,10 +85,6 @@ You can update this setting through the
 
 Sets the timeout for collecting index statistics. Defaults to `10s`.
 
-`xpack.monitoring.collection.indices.stats.timeout`::
-
-Sets the timeout for collecting total indices statistics. Defaults to `10s`.
-
 `xpack.monitoring.collection.index.recovery.active_only`::
 
 Controls whether or not all recoveries are collected. Set to `true` to

From ac960bfa6b6bd96e83cd186bf95667f48be1e0e6 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Fri, 20 Jul 2018 15:49:37 +0100
Subject: [PATCH 115/260] [ML] Use default request durability for .ml-state
 index (#32233)

The initial decision to use async durability was made a long time ago
for performance reasons. That argument no longer applies and we
prefer the safety of request durability.
---
 .../java/org/elasticsearch/xpack/ml/MachineLearning.java    | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 3d1011c47e2..d5461d85599 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -635,11 +635,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
                         // TODO review these settings
                         .settings(Settings.builder()
                                 .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
-                                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting)
-                                // Sacrifice durability for performance: in the event of power
-                                // failure we can lose the last 5 seconds of changes, but it's
-                                // much faster
-                                .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "async"))
+                                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting))
                         .putMapping(ElasticsearchMappings.DOC_TYPE, Strings.toString(stateMapping))
                         .version(Version.CURRENT.id)
                         .build();

From 74aa7b0815c048271a5536aad7941ca72a0d8f6c Mon Sep 17 00:00:00 2001
From: Lee Hinman <dakrone@users.noreply.github.com>
Date: Fri, 20 Jul 2018 08:52:45 -0600
Subject: [PATCH 116/260] Enhance Parent circuit breaker error message (#32056)

* Enhance Parent circuit breaker error message

This adds information about either the current real usage (if tracking "real"
memory usage) or the child breaker usages to the exception message when the
parent circuit breaker trips.

The messages now look like:

```
[parent] Data too large, data for [my_request] would be [211288064/201.5mb], which is larger than the limit of [209715200/200mb], usages [request=157286400/150mb, fielddata=54001664/51.5mb, in_flight_requests=0/0b, accounting=0/0b]
```

Or when tracking real memory usage:

```
[parent] Data too large, data for [request] would be [251/251b], which is larger than the limit of [200/200b], real usage: [181/181b], new bytes reserved: [70/70b]
```

* Only call currentMemoryUsage once by returning structured object
---
 .../HierarchyCircuitBreakerService.java       | 54 +++++++++++++++----
 .../HierarchyCircuitBreakerServiceTests.java  |  5 ++
 2 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
index ebebcfd253c..7e6a9c29a83 100644
--- a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
+++ b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
@@ -37,6 +37,7 @@ import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
 
 /**
  * CircuitBreakerService that attempts to redistribute space between breakers
@@ -215,7 +216,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
         }
         // Manually add the parent breaker settings since they aren't part of the breaker map
         allStats.add(new CircuitBreakerStats(CircuitBreaker.PARENT, parentSettings.getLimit(),
-            parentUsed(0L), 1.0, parentTripCount.get()));
+            parentUsed(0L).totalUsage, 1.0, parentTripCount.get()));
         return new AllCircuitBreakerStats(allStats.toArray(new CircuitBreakerStats[allStats.size()]));
     }
 
@@ -225,15 +226,26 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
         return new CircuitBreakerStats(breaker.getName(), breaker.getLimit(), breaker.getUsed(), breaker.getOverhead(), breaker.getTrippedCount());
     }
 
-    private long parentUsed(long newBytesReserved) {
+    private static class ParentMemoryUsage {
+        final long baseUsage;
+        final long totalUsage;
+
+        ParentMemoryUsage(final long baseUsage, final long totalUsage) {
+            this.baseUsage = baseUsage;
+            this.totalUsage = totalUsage;
+        }
+    }
+
+    private ParentMemoryUsage parentUsed(long newBytesReserved) {
         if (this.trackRealMemoryUsage) {
-            return currentMemoryUsage() + newBytesReserved;
+            final long current = currentMemoryUsage();
+            return new ParentMemoryUsage(current, current + newBytesReserved);
         } else {
             long parentEstimated = 0;
             for (CircuitBreaker breaker : this.breakers.values()) {
                 parentEstimated += breaker.getUsed() * breaker.getOverhead();
             }
-            return parentEstimated;
+            return new ParentMemoryUsage(parentEstimated, parentEstimated);
         }
     }
 
@@ -246,15 +258,37 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
      * Checks whether the parent breaker has been tripped
      */
     public void checkParentLimit(long newBytesReserved, String label) throws CircuitBreakingException {
-        long totalUsed = parentUsed(newBytesReserved);
+        final ParentMemoryUsage parentUsed = parentUsed(newBytesReserved);
         long parentLimit = this.parentSettings.getLimit();
-        if (totalUsed > parentLimit) {
+        if (parentUsed.totalUsage > parentLimit) {
             this.parentTripCount.incrementAndGet();
-            final String message = "[parent] Data too large, data for [" + label + "]" +
-                    " would be [" + totalUsed + "/" + new ByteSizeValue(totalUsed) + "]" +
+            final StringBuilder message = new StringBuilder("[parent] Data too large, data for [" + label + "]" +
+                    " would be [" + parentUsed.totalUsage + "/" + new ByteSizeValue(parentUsed.totalUsage) + "]" +
                     ", which is larger than the limit of [" +
-                    parentLimit + "/" + new ByteSizeValue(parentLimit) + "]";
-            throw new CircuitBreakingException(message, totalUsed, parentLimit);
+                    parentLimit + "/" + new ByteSizeValue(parentLimit) + "]");
+            if (this.trackRealMemoryUsage) {
+                final long realUsage = parentUsed.baseUsage;
+                message.append(", real usage: [");
+                message.append(realUsage);
+                message.append("/");
+                message.append(new ByteSizeValue(realUsage));
+                message.append("], new bytes reserved: [");
+                message.append(newBytesReserved);
+                message.append("/");
+                message.append(new ByteSizeValue(newBytesReserved));
+                message.append("]");
+            } else {
+                message.append(", usages [");
+                message.append(String.join(", ",
+                    this.breakers.entrySet().stream().map(e -> {
+                        final CircuitBreaker breaker = e.getValue();
+                        final long breakerUsed = (long)(breaker.getUsed() * breaker.getOverhead());
+                        return e.getKey() + "=" + breakerUsed + "/" + new ByteSizeValue(breakerUsed);
+                    })
+                        .collect(Collectors.toList())));
+                message.append("]");
+            }
+            throw new CircuitBreakingException(message.toString(), parentUsed.totalUsage, parentLimit);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
index a73cf8630fe..6f8689a9664 100644
--- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
@@ -199,6 +199,8 @@ public class HierarchyCircuitBreakerServiceTests extends ESTestCase {
                 .addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should break"));
             assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [should break] would be"));
             assertThat(exception.getMessage(), containsString("which is larger than the limit of [209715200/200mb]"));
+            assertThat(exception.getMessage(),
+                containsString("usages [request=157286400/150mb, fielddata=54001664/51.5mb, in_flight_requests=0/0b, accounting=0/0b]"));
         }
     }
 
@@ -239,6 +241,9 @@ public class HierarchyCircuitBreakerServiceTests extends ESTestCase {
         // it was the parent that rejected the reservation
         assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [request] would be"));
         assertThat(exception.getMessage(), containsString("which is larger than the limit of [200/200b]"));
+        assertThat(exception.getMessage(),
+            containsString("real usage: [181/181b], new bytes reserved: [" + (reservationInBytes * 2) +
+                "/" + new ByteSizeValue(reservationInBytes * 2) + "]"));
         assertEquals(0, requestBreaker.getTrippedCount());
         assertEquals(1, service.stats().getStats(CircuitBreaker.PARENT).getTrippedCount());
 

From 6ed1ad0b6ffd5eba5be0807d7ea96aefe722840f Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 20 Jul 2018 16:55:11 +0200
Subject: [PATCH 117/260] Fix multi level nested sort (#32204)

The parent filter for nested sort should always match **all** parents regardless
of the child queries. It is used to find the boundaries of a single parent and we use
the child query to match all the filters set in the nested tree so there is no need to
repeat the nested filters.
With this change we ensure that we build bitset filters
only to find the root docs (or the docs at the level where the sort applies) that can be reused
among queries.

Closes #31554
Closes #32130
Closes #31783

Co-authored-by: Dominic Bevacqua <bev@treatwell.com>
---
 .../search/sort/SortBuilder.java              |  39 +-
 .../search/nested/NestedSortingTests.java     | 453 ++++++++++++++++++
 .../search/nested/SimpleNestedIT.java         |  73 +++
 3 files changed, 544 insertions(+), 21 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
index 9537e288919..9d2a5c9f1e2 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
@@ -22,9 +22,7 @@ package org.elasticsearch.search.sort;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.search.join.ToChildBlockJoinQuery;
-import org.apache.lucene.search.join.ToParentBlockJoinQuery;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -186,10 +184,21 @@ public abstract class SortBuilder<T extends SortBuilder<T>> implements NamedWrit
     }
 
     protected static Nested resolveNested(QueryShardContext context, NestedSortBuilder nestedSort) throws IOException {
-        return resolveNested(context, nestedSort, null);
+        final Query childQuery = resolveNestedQuery(context, nestedSort, null);
+        if (childQuery == null) {
+            return null;
+        }
+        final ObjectMapper objectMapper = context.nestedScope().getObjectMapper();
+        final Query parentQuery;
+        if (objectMapper == null) {
+            parentQuery = Queries.newNonNestedFilter(context.indexVersionCreated());
+        } else {
+            parentQuery = objectMapper.nestedTypeFilter();
+        }
+        return new Nested(context.bitsetFilter(parentQuery), childQuery);
     }
 
-    private static Nested resolveNested(QueryShardContext context, NestedSortBuilder nestedSort, Nested nested) throws IOException {
+    private static Query resolveNestedQuery(QueryShardContext context, NestedSortBuilder nestedSort, Query parentQuery) throws IOException {
         if (nestedSort == null || nestedSort.getPath() == null) {
             return null;
         }
@@ -207,15 +216,7 @@ public abstract class SortBuilder<T extends SortBuilder<T>> implements NamedWrit
         if (!nestedObjectMapper.nested().isNested()) {
             throw new QueryShardException(context, "[nested] nested object under path [" + nestedPath + "] is not of nested type");
         }
-
-        // get our parent query which will determines our parent documents
-        Query parentQuery;
         ObjectMapper objectMapper = context.nestedScope().getObjectMapper();
-        if (objectMapper == null) {
-            parentQuery = Queries.newNonNestedFilter(context.indexVersionCreated());
-        } else {
-            parentQuery = objectMapper.nestedTypeFilter();
-        }
 
         // get our child query, potentially applying a users filter
         Query childQuery;
@@ -223,7 +224,7 @@ public abstract class SortBuilder<T extends SortBuilder<T>> implements NamedWrit
             context.nestedScope().nextLevel(nestedObjectMapper);
             if (nestedFilter != null) {
                 assert nestedFilter == Rewriteable.rewrite(nestedFilter, context) : "nested filter is not rewritten";
-                if (nested == null) {
+                if (parentQuery == null) {
                     // this is for back-compat, original single level nested sorting never applied a nested type filter
                     childQuery = nestedFilter.toFilter(context);
                 } else {
@@ -237,27 +238,23 @@ public abstract class SortBuilder<T extends SortBuilder<T>> implements NamedWrit
         }
 
         // apply filters from the previous nested level
-        if (nested != null) {
-            parentQuery = Queries.filtered(parentQuery,
-                new ToParentBlockJoinQuery(nested.getInnerQuery(), nested.getRootFilter(), ScoreMode.None));
-
+        if (parentQuery != null) {
             if (objectMapper != null) {
                 childQuery = Queries.filtered(childQuery,
-                    new ToChildBlockJoinQuery(nested.getInnerQuery(), context.bitsetFilter(objectMapper.nestedTypeFilter())));
+                    new ToChildBlockJoinQuery(parentQuery, context.bitsetFilter(objectMapper.nestedTypeFilter())));
             }
         }
 
         // wrap up our parent and child and either process the next level of nesting or return
-        final Nested innerNested = new Nested(context.bitsetFilter(parentQuery), childQuery);
         if (nestedNestedSort != null) {
             try {
                 context.nestedScope().nextLevel(nestedObjectMapper);
-                return resolveNested(context, nestedNestedSort, innerNested);
+                return resolveNestedQuery(context, nestedNestedSort, childQuery);
             } finally {
                 context.nestedScope().previousLevel();
             }
         } else {
-            return innerNested;
+            return childQuery;
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
index 57f2310e877..1300debd5eb 100644
--- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
@@ -21,7 +21,11 @@ package org.elasticsearch.index.search.nested;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause.Occur;
@@ -40,21 +44,37 @@ import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.search.join.ToParentBlockJoinQuery;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.fielddata.AbstractFieldDataTestCase;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.NoOrdinalsStringFieldDataTests;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.NestedQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.NestedSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME;
 import static org.hamcrest.Matchers.equalTo;
 
 public class NestedSortingTests extends AbstractFieldDataTestCase {
@@ -343,4 +363,437 @@ public class NestedSortingTests extends AbstractFieldDataTestCase {
         searcher.getIndexReader().close();
     }
 
+    public void testMultiLevelNestedSorting() throws IOException {
+        XContentBuilder mapping = XContentFactory.jsonBuilder();
+        mapping.startObject();
+        {
+            mapping.startObject("_doc");
+            {
+                mapping.startObject("properties");
+                {
+                    {
+                        mapping.startObject("title");
+                        mapping.field("type", "text");
+                        mapping.endObject();
+                    }
+                    {
+                        mapping.startObject("genre");
+                        mapping.field("type", "keyword");
+                        mapping.endObject();
+                    }
+                    {
+                        mapping.startObject("chapters");
+                        mapping.field("type", "nested");
+                        {
+                            mapping.startObject("properties");
+                            {
+                                mapping.startObject("title");
+                                mapping.field("type", "text");
+                                mapping.endObject();
+                            }
+                            {
+                                mapping.startObject("read_time_seconds");
+                                mapping.field("type", "integer");
+                                mapping.endObject();
+                            }
+                            {
+                                mapping.startObject("paragraphs");
+                                mapping.field("type", "nested");
+                                {
+                                    mapping.startObject("properties");
+                                    {
+                                        {
+                                            mapping.startObject("header");
+                                            mapping.field("type", "text");
+                                            mapping.endObject();
+                                        }
+                                        {
+                                            mapping.startObject("content");
+                                            mapping.field("type", "text");
+                                            mapping.endObject();
+                                        }
+                                        {
+                                            mapping.startObject("word_count");
+                                            mapping.field("type", "integer");
+                                            mapping.endObject();
+                                        }
+                                    }
+                                    mapping.endObject();
+                                }
+                                mapping.endObject();
+                            }
+                            mapping.endObject();
+                        }
+                        mapping.endObject();
+                    }
+                }
+                mapping.endObject();
+            }
+            mapping.endObject();
+        }
+        mapping.endObject();
+        IndexService indexService = createIndex("nested_sorting", Settings.EMPTY, "_doc", mapping);
+
+        List<List<Document>> books = new ArrayList<>();
+        {
+            List<Document> book = new ArrayList<>();
+            Document document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "Paragraph 1", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 743));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 743));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.title", "chapter 3", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters", Field.Store.NO));
+            document.add(new IntPoint("chapters.read_time_seconds", 400));
+            document.add(new NumericDocValuesField("chapters.read_time_seconds", 400));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "Paragraph 1", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 234));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 234));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.title", "chapter 2", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters", Field.Store.NO));
+            document.add(new IntPoint("chapters.read_time_seconds", 200));
+            document.add(new NumericDocValuesField("chapters.read_time_seconds", 200));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "Paragraph 2", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 478));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 478));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "Paragraph 1", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 849));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 849));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.title", "chapter 1", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters", Field.Store.NO));
+            document.add(new IntPoint("chapters.read_time_seconds", 1400));
+            document.add(new NumericDocValuesField("chapters.read_time_seconds", 1400));
+            book.add(document);
+            document = new Document();
+            document.add(new StringField("genre", "science fiction", Field.Store.NO));
+            document.add(new StringField("_type", "_doc", Field.Store.NO));
+            document.add(new StringField("_id", "1", Field.Store.YES));
+            document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0));
+            book.add(document);
+            books.add(book);
+        }
+        {
+            List<Document> book = new ArrayList<>();
+            Document document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "Introduction", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 76));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 76));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.title", "chapter 1", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters", Field.Store.NO));
+            document.add(new IntPoint("chapters.read_time_seconds", 20));
+            document.add(new NumericDocValuesField("chapters.read_time_seconds", 20));
+            book.add(document);
+            document = new Document();
+            document.add(new StringField("genre", "romance", Field.Store.NO));
+            document.add(new StringField("_type", "_doc", Field.Store.NO));
+            document.add(new StringField("_id", "2", Field.Store.YES));
+            document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0));
+            book.add(document);
+            books.add(book);
+        }
+        {
+            List<Document> book = new ArrayList<>();
+            Document document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "A bad dream", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 976));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 976));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.title", "The beginning of the end", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters", Field.Store.NO));
+            document.add(new IntPoint("chapters.read_time_seconds", 1200));
+            document.add(new NumericDocValuesField("chapters.read_time_seconds", 1200));
+            book.add(document);
+            document = new Document();
+            document.add(new StringField("genre", "horror", Field.Store.NO));
+            document.add(new StringField("_type", "_doc", Field.Store.NO));
+            document.add(new StringField("_id", "3", Field.Store.YES));
+            document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0));
+            book.add(document);
+            books.add(book);
+        }
+        {
+            List<Document> book = new ArrayList<>();
+            Document document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "macaroni", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 180));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 180));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "hamburger", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 150));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 150));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "tosti", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 120));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 120));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.title", "easy meals", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters", Field.Store.NO));
+            document.add(new IntPoint("chapters.read_time_seconds", 800));
+            document.add(new NumericDocValuesField("chapters.read_time_seconds", 800));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.paragraphs.header", "introduction", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters.paragraphs", Field.Store.NO));
+            document.add(new TextField("chapters.paragraphs.text", "some text...", Field.Store.NO));
+            document.add(new SortedNumericDocValuesField("chapters.paragraphs.word_count", 87));
+            document.add(new IntPoint("chapters.paragraphs.word_count", 87));
+            book.add(document);
+            document = new Document();
+            document.add(new TextField("chapters.title", "introduction", Field.Store.NO));
+            document.add(new StringField("_type", "__chapters", Field.Store.NO));
+            document.add(new IntPoint("chapters.read_time_seconds", 10));
+            document.add(new NumericDocValuesField("chapters.read_time_seconds", 10));
+            book.add(document);
+            document = new Document();
+            document.add(new StringField("genre", "cooking", Field.Store.NO));
+            document.add(new StringField("_type", "_doc", Field.Store.NO));
+            document.add(new StringField("_id", "4", Field.Store.YES));
+            document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0));
+            book.add(document);
+            books.add(book);
+        }
+        {
+            List<Document> book = new ArrayList<>();
+            Document document = new Document();
+            document.add(new StringField("genre", "unknown", Field.Store.NO));
+            document.add(new StringField("_type", "_doc", Field.Store.NO));
+            document.add(new StringField("_id", "5", Field.Store.YES));
+            document.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0));
+            book.add(document);
+            books.add(book);
+        }
+
+        Collections.shuffle(books, random());
+        for (List<Document> book : books) {
+            writer.addDocuments(book);
+            if (randomBoolean()) {
+                writer.commit();
+            }
+        }
+        DirectoryReader reader = DirectoryReader.open(writer);
+        reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0));
+        IndexSearcher searcher = new IndexSearcher(reader);
+        QueryShardContext queryShardContext = indexService.newQueryShardContext(0, reader, () -> 0L, null);
+
+        FieldSortBuilder sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count");
+        sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs")));
+        QueryBuilder queryBuilder = new MatchAllQueryBuilder();
+        TopFieldDocs topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+        assertThat(topFields.totalHits, equalTo(5L));
+        assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+        assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
+        assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("4"));
+        assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(87L));
+        assertThat(searcher.doc(topFields.scoreDocs[2].doc).get("_id"), equalTo("1"));
+        assertThat(((FieldDoc) topFields.scoreDocs[2]).fields[0], equalTo(234L));
+        assertThat(searcher.doc(topFields.scoreDocs[3].doc).get("_id"), equalTo("3"));
+        assertThat(((FieldDoc) topFields.scoreDocs[3]).fields[0], equalTo(976L));
+        assertThat(searcher.doc(topFields.scoreDocs[4].doc).get("_id"), equalTo("5"));
+        assertThat(((FieldDoc) topFields.scoreDocs[4]).fields[0], equalTo(Long.MAX_VALUE));
+
+        // Specific genre
+        {
+            queryBuilder = new TermQueryBuilder("genre", "romance");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
+
+            queryBuilder = new TermQueryBuilder("genre", "science fiction");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(234L));
+
+            queryBuilder = new TermQueryBuilder("genre", "horror");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(976L));
+
+            queryBuilder = new TermQueryBuilder("genre", "cooking");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
+        }
+
+        // reverse sort order
+        {
+            sortBuilder.order(SortOrder.DESC);
+            queryBuilder = new MatchAllQueryBuilder();
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(5L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(976L));
+            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("1"));
+            assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(849L));
+            assertThat(searcher.doc(topFields.scoreDocs[2].doc).get("_id"), equalTo("4"));
+            assertThat(((FieldDoc) topFields.scoreDocs[2]).fields[0], equalTo(180L));
+            assertThat(searcher.doc(topFields.scoreDocs[3].doc).get("_id"), equalTo("2"));
+            assertThat(((FieldDoc) topFields.scoreDocs[3]).fields[0], equalTo(76L));
+            assertThat(searcher.doc(topFields.scoreDocs[4].doc).get("_id"), equalTo("5"));
+            assertThat(((FieldDoc) topFields.scoreDocs[4]).fields[0], equalTo(Long.MIN_VALUE));
+        }
+
+        // Specific genre and reverse sort order
+        {
+            queryBuilder = new TermQueryBuilder("genre", "romance");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
+
+            queryBuilder = new TermQueryBuilder("genre", "science fiction");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(849L));
+
+            queryBuilder = new TermQueryBuilder("genre", "horror");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(976L));
+
+            queryBuilder = new TermQueryBuilder("genre", "cooking");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(180L));
+        }
+
+        // Nested filter + query
+        {
+            queryBuilder = new RangeQueryBuilder("chapters.read_time_seconds").to(50L);
+            sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count");
+            sortBuilder.setNestedSort(
+                new NestedSortBuilder("chapters")
+                    .setFilter(queryBuilder)
+                    .setNestedSort(new NestedSortBuilder("chapters.paragraphs"))
+            );
+            topFields = search(new NestedQueryBuilder("chapters", queryBuilder, ScoreMode.None), sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(2L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
+            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("4"));
+            assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(87L));
+
+            sortBuilder.order(SortOrder.DESC);
+            topFields = search(new NestedQueryBuilder("chapters", queryBuilder, ScoreMode.None), sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(2L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
+            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
+            assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(76L));
+        }
+
+        // Multiple Nested filters + query
+        {
+            queryBuilder = new RangeQueryBuilder("chapters.read_time_seconds").to(50L);
+            sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count");
+            sortBuilder.setNestedSort(
+                new NestedSortBuilder("chapters")
+                    .setFilter(queryBuilder)
+                    .setNestedSort(
+                        new NestedSortBuilder("chapters.paragraphs")
+                            .setFilter(new RangeQueryBuilder("chapters.paragraphs.word_count").from(80L))
+                    )
+            );
+            topFields = search(new NestedQueryBuilder("chapters", queryBuilder, ScoreMode.None), sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(2L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
+            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
+            assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(Long.MAX_VALUE));
+
+            sortBuilder.order(SortOrder.DESC);
+            topFields = search(new NestedQueryBuilder("chapters", queryBuilder, ScoreMode.None), sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(2L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
+            assertThat(searcher.doc(topFields.scoreDocs[1].doc).get("_id"), equalTo("2"));
+            assertThat(((FieldDoc) topFields.scoreDocs[1]).fields[0], equalTo(Long.MIN_VALUE));
+        }
+
+        // Nested filter + Specific genre
+        {
+            sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count");
+            sortBuilder.setNestedSort(
+                new NestedSortBuilder("chapters")
+                    .setFilter(new RangeQueryBuilder("chapters.read_time_seconds").to(50L))
+                    .setNestedSort(new NestedSortBuilder("chapters.paragraphs"))
+            );
+
+            queryBuilder = new TermQueryBuilder("genre", "romance");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("2"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(76L));
+
+            queryBuilder = new TermQueryBuilder("genre", "science fiction");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("1"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(Long.MAX_VALUE));
+
+            queryBuilder = new TermQueryBuilder("genre", "horror");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("3"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(Long.MAX_VALUE));
+
+            queryBuilder = new TermQueryBuilder("genre", "cooking");
+            topFields = search(queryBuilder, sortBuilder, queryShardContext, searcher);
+            assertThat(topFields.totalHits, equalTo(1L));
+            assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
+            assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
+        }
+    }
+
+    private static TopFieldDocs search(QueryBuilder queryBuilder, FieldSortBuilder sortBuilder, QueryShardContext queryShardContext,
+                                       IndexSearcher searcher) throws IOException {
+        Query query = new BooleanQuery.Builder()
+            .add(queryBuilder.toQuery(queryShardContext), Occur.MUST)
+            .add(Queries.newNonNestedFilter(Version.CURRENT), Occur.FILTER)
+            .build();
+        Sort sort = new Sort(sortBuilder.build(queryShardContext).field);
+        return searcher.search(query, 10, sort);
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/server/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index 68ef78f4273..6d8bcfb6131 100644
--- a/server/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/server/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -709,6 +709,79 @@ public class SimpleNestedIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("1")); // missing last
     }
 
+    // https://github.com/elastic/elasticsearch/issues/31554
+    public void testLeakingSortValues() throws Exception {
+        assertAcked(prepareCreate("test")
+            .setSettings(Settings.builder().put("number_of_shards", 1))
+            .addMapping("test-type", "{\n"
+                    + "        \"dynamic\": \"strict\",\n"
+                    + "        \"properties\": {\n"
+                    + "          \"nested1\": {\n"
+                    + "            \"type\": \"nested\",\n"
+                    + "            \"properties\": {\n"
+                    + "              \"nested2\": {\n"
+                    + "                \"type\": \"nested\",\n"
+                    + "                \"properties\": {\n"
+                    + "                  \"nested2_keyword\": {\n"
+                    + "                    \"type\": \"keyword\"\n"
+                    + "                  },\n"
+                    + "                  \"sortVal\": {\n"
+                    + "                    \"type\": \"integer\"\n"
+                    + "                  }\n"
+                    + "                }\n"
+                    + "              }\n"
+                    + "            }\n"
+                    + "          }\n"
+                    + "        }\n"
+                    + "      }\n", XContentType.JSON));
+            ensureGreen();
+
+            client().prepareIndex("test", "test-type", "1").setSource("{\n"
+                    + "  \"nested1\": [\n"
+                    + "    {\n"
+                    + "      \"nested2\": [\n"
+                    + "        {\n"
+                    + "          \"nested2_keyword\": \"nested2_bar\",\n"
+                    + "          \"sortVal\": 1\n"
+                    + "        }\n"
+                    + "      ]\n"
+                    + "    }\n"
+                    + " ]\n"
+                    + "}", XContentType.JSON).execute().actionGet();
+
+            client().prepareIndex("test", "test-type", "2").setSource("{\n"
+                    + "  \"nested1\": [\n"
+                    + "    {\n"
+                    + "      \"nested2\": [\n"
+                    + "        {\n"
+                    + "          \"nested2_keyword\": \"nested2_bar\",\n"
+                    + "          \"sortVal\": 2\n"
+                    + "        }\n"
+                    + "      ]\n"
+                    + "    } \n"
+                    + "  ]\n"
+                    + "}", XContentType.JSON).execute().actionGet();
+
+            refresh();
+
+            SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(termQuery("_id", 2))
+                .addSort(
+                    SortBuilders
+                        .fieldSort("nested1.nested2.sortVal")
+                        .setNestedSort(new NestedSortBuilder("nested1")
+                        .setNestedSort(new NestedSortBuilder("nested1.nested2")
+                        .setFilter(termQuery("nested1.nested2.nested2_keyword", "nested2_bar"))))
+                )
+                .execute().actionGet();
+
+            assertHitCount(searchResponse, 1);
+            assertThat(searchResponse.getHits().getHits().length, equalTo(1));
+            assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("2"));
+            assertThat(searchResponse.getHits().getHits()[0].getSortValues()[0].toString(), equalTo("2"));
+
+    }
+
     public void testSortNestedWithNestedFilter() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", XContentFactory.jsonBuilder()

From 91a0daf0e49f21e5dc909ad395b3eb8fad4e9075 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 20 Jul 2018 17:14:17 +0200
Subject: [PATCH 118/260] MINOR: Remove unused `IndexDynamicSettings` (#32237)

---
 .../index/settings/IndexDynamicSettings.java  | 39 -------------------
 1 file changed, 39 deletions(-)
 delete mode 100644 server/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettings.java

diff --git a/server/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettings.java b/server/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettings.java
deleted file mode 100644
index 0f686efe926..00000000000
--- a/server/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettings.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.settings;
-
-import org.elasticsearch.common.inject.BindingAnnotation;
-
-import java.lang.annotation.Documented;
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-
-@BindingAnnotation
-@Target({FIELD, PARAMETER})
-@Retention(RUNTIME)
-@Documented
-public @interface IndexDynamicSettings {
-
-}

From 320f1d263f51f7b6c36158b6774a05844dcab86e Mon Sep 17 00:00:00 2001
From: Paul Sanwald <paul.sanwald@elastic.co>
Date: Fri, 20 Jul 2018 11:20:51 -0400
Subject: [PATCH 119/260] muting failing test for internal auto date histogram
 to avoid failure before fix is merged

---
 .../bucket/histogram/InternalAutoDateHistogramTests.java    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
index 389371efd79..96811ce424c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
@@ -117,6 +117,12 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati
         return ParsedAutoDateHistogram.class;
     }
 
+    @Override
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32215")
+    public void testReduceRandom() {
+        super.testReduceRandom();
+    }
+
     @Override
     protected InternalAutoDateHistogram mutateInstance(InternalAutoDateHistogram instance) {
         String name = instance.getName();

From c6c9075ca49af7a1aebdcbb01aa36c5adcc4a55f Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 20 Jul 2018 12:01:50 -0400
Subject: [PATCH 120/260] Switch rolling restart to new style Requests (#32147)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `qa/rolling-upgrade` project to use the new
versions.
---
 .../elasticsearch/upgrades/RecoveryIT.java    | 33 +++++++++----------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
index 809cd40d698..de5681ebe1a 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.upgrades;
 
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Request;
@@ -32,14 +30,12 @@ import org.elasticsearch.test.rest.yaml.ObjectPath;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Future;
 import java.util.function.Predicate;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength;
-import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
@@ -65,8 +61,9 @@ public class RecoveryIT extends AbstractRollingTestCase {
             createIndex(index, settings.build());
         } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
             ensureGreen(index);
-            Response response = client().performRequest("GET", index + "/_stats", Collections.singletonMap("level", "shards"));
-            assertOK(response);
+            Request shardStatsRequest = new Request("GET", index + "/_stats");
+            shardStatsRequest.addParameter("level", "shards");
+            Response response = client().performRequest(shardStatsRequest);
             ObjectPath objectPath = ObjectPath.createFromResponse(response);
             List<Object> shardStats = objectPath.evaluate("indices." + index + ".shards.0");
             assertThat(shardStats, hasSize(2));
@@ -87,8 +84,9 @@ public class RecoveryIT extends AbstractRollingTestCase {
     private int indexDocs(String index, final int idStart, final int numDocs) throws IOException {
         for (int i = 0; i < numDocs; i++) {
             final int id = idStart + i;
-            assertOK(client().performRequest("PUT", index + "/test/" + id, emptyMap(),
-                new StringEntity("{\"test\": \"test_" + randomAsciiOfLength(2) + "\"}", ContentType.APPLICATION_JSON)));
+            Request indexDoc = new Request("PUT", index + "/test/" + id);
+            indexDoc.setJsonEntity("{\"test\": \"test_" + randomAsciiOfLength(2) + "\"}");
+            client().performRequest(indexDoc);
         }
         return numDocs;
     }
@@ -113,7 +111,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
 
     public void testRecoveryWithConcurrentIndexing() throws Exception {
         final String index = "recovery_with_concurrent_indexing";
-        Response response = client().performRequest("GET", "_nodes");
+        Response response = client().performRequest(new Request("GET", "_nodes"));
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
         final Map<String, Object> nodeMap = objectPath.evaluate("nodes");
         List<String> nodes = new ArrayList<>(nodeMap.keySet());
@@ -139,7 +137,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
                 updateIndexSettings(index, Settings.builder().put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null));
                 asyncIndexDocs(index, 10, 50).get();
                 ensureGreen(index);
-                assertOK(client().performRequest("POST", index + "/_refresh"));
+                client().performRequest(new Request("POST", index + "/_refresh"));
                 assertCount(index, "_only_nodes:" + nodes.get(0), 60);
                 assertCount(index, "_only_nodes:" + nodes.get(1), 60);
                 assertCount(index, "_only_nodes:" + nodes.get(2), 60);
@@ -150,7 +148,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
                 updateIndexSettings(index, Settings.builder().put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null));
                 asyncIndexDocs(index, 60, 50).get();
                 ensureGreen(index);
-                assertOK(client().performRequest("POST", index + "/_refresh"));
+                client().performRequest(new Request("POST", index + "/_refresh"));
                 assertCount(index, "_only_nodes:" + nodes.get(0), 110);
                 assertCount(index, "_only_nodes:" + nodes.get(1), 110);
                 assertCount(index, "_only_nodes:" + nodes.get(2), 110);
@@ -161,15 +159,16 @@ public class RecoveryIT extends AbstractRollingTestCase {
     }
 
     private void assertCount(final String index, final String preference, final int expectedCount) throws IOException {
-        final Response response = client().performRequest("GET", index + "/_count", Collections.singletonMap("preference", preference));
-        assertOK(response);
+        final Request request = new Request("GET", index + "/_count");
+        request.addParameter("preference", preference);
+        final Response response = client().performRequest(request);
         final int actualCount = Integer.parseInt(ObjectPath.createFromResponse(response).evaluate("count").toString());
         assertThat(actualCount, equalTo(expectedCount));
     }
 
 
     private String getNodeId(Predicate<Version> versionPredicate) throws IOException {
-        Response response = client().performRequest("GET", "_nodes");
+        Response response = client().performRequest(new Request("GET", "_nodes"));
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
         Map<String, Object> nodesAsMap = objectPath.evaluate("nodes");
         for (String id : nodesAsMap.keySet()) {
@@ -216,7 +215,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
                 updateIndexSettings(index, Settings.builder().put("index.routing.allocation.include._id", newNode));
                 asyncIndexDocs(index, 10, 50).get();
                 ensureGreen(index);
-                assertOK(client().performRequest("POST", index + "/_refresh"));
+                client().performRequest(new Request("POST", index + "/_refresh"));
                 assertCount(index, "_only_nodes:" + newNode, 60);
                 break;
             case UPGRADED:
@@ -226,8 +225,8 @@ public class RecoveryIT extends AbstractRollingTestCase {
                 );
                 asyncIndexDocs(index, 60, 50).get();
                 ensureGreen(index);
-                assertOK(client().performRequest("POST", index + "/_refresh"));
-                Response response = client().performRequest("GET", "_nodes");
+                client().performRequest(new Request("POST", index + "/_refresh"));
+                Response response = client().performRequest(new Request("GET", "_nodes"));
                 ObjectPath objectPath = ObjectPath.createFromResponse(response);
                 final Map<String, Object> nodeMap = objectPath.evaluate("nodes");
                 List<String> nodes = new ArrayList<>(nodeMap.keySet());

From af0c1d30fe8570d33cd413b6a81e737ecd754524 Mon Sep 17 00:00:00 2001
From: Julie Tibshirani <julie.tibshirani@elastic.co>
Date: Fri, 20 Jul 2018 10:06:07 -0700
Subject: [PATCH 121/260] Make sure that field aliases count towards the total
 fields limit. (#32222)

---
 .../index/mapper/MapperService.java           |  2 +-
 .../index/mapper/MapperServiceTests.java      | 31 +++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 936e7334002..921e472c94f 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -427,7 +427,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
                 // the master node restoring mappings from disk or data nodes
                 // deserializing cluster state that was sent by the master node,
                 // this check will be skipped.
-                checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
+                checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size() + fieldAliasMappers.size());
             }
 
             results.put(newMapper.type(), newMapper);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index 20e0dd4639c..51b6e9d7168 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -270,6 +270,37 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
         assertThat(e.getMessage(), containsString("Invalid [path] value [nested.field] for field alias [alias]"));
     }
 
+    public void testTotalFieldsLimitWithFieldAlias() throws Throwable {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties")
+                .startObject("alias")
+                    .field("type", "alias")
+                    .field("path", "field")
+                .endObject()
+                .startObject("field")
+                    .field("type", "text")
+                .endObject()
+            .endObject()
+        .endObject().endObject());
+
+        DocumentMapper documentMapper = createIndex("test1").mapperService()
+            .merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
+
+        // Set the total fields limit to the number of non-alias fields, to verify that adding
+        // a field alias pushes the mapping over the limit.
+        int numFields = documentMapper.mapping().metadataMappers.length + 2;
+        int numNonAliasFields = numFields - 1;
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            Settings settings = Settings.builder()
+                .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numNonAliasFields)
+                .build();
+            createIndex("test2", settings).mapperService()
+                .merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
+        });
+        assertEquals("Limit of total fields [" + numNonAliasFields + "] in index [test2] has been exceeded", e.getMessage());
+    }
+
     public void testForbidMultipleTypes() throws IOException {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
         MapperService mapperService = createIndex("test").mapperService();

From 54d896c4ede28500a01e308be7c0f504a1835b63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Fri, 20 Jul 2018 19:08:59 +0200
Subject: [PATCH 122/260] [Tests] Remove QueryStringQueryBuilderTests#toQuery
 class assertions (#32236)

Currently we check that the queries that QueryStringQueryBuilder#toQuery returns
is one out of a list of many Lucene query classes. This list has extended a lot over time,
since QueryStringQueryBuilder can build all sort of queries. This makes the test hard to
maintain. The recent addition of alias fields which build a BlendedTermQuery show how
easy this test breaks. Also the current assertions doesn't add a lot in terms of catching
errors. This is why we decided to remove this check.

Closes #32234
---
 .../index/query/QueryStringQueryBuilderTests.java        | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index 4b9e0f5a66e..591ee5af080 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -42,7 +42,6 @@ import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanOrQuery;
-import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
@@ -51,7 +50,6 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -74,7 +72,6 @@ import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQuery
 import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertDisjunctionSubQuery;
-import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
@@ -381,11 +378,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     @Override
     protected void doAssertLuceneQuery(QueryStringQueryBuilder queryBuilder,
                                        Query query, SearchContext context) throws IOException {
-        assertThat(query, either(instanceOf(TermQuery.class))
-            .or(instanceOf(BooleanQuery.class)).or(instanceOf(DisjunctionMaxQuery.class))
-            .or(instanceOf(PhraseQuery.class)).or(instanceOf(BoostQuery.class))
-            .or(instanceOf(MultiPhrasePrefixQuery.class)).or(instanceOf(PrefixQuery.class)).or(instanceOf(SpanQuery.class))
-            .or(instanceOf(MatchNoDocsQuery.class)));
+        // nothing yet, put additional assertions here.
     }
 
     // Tests fix for https://github.com/elastic/elasticsearch/issues/29403

From ac6340865560edd221e634b541312d4ed3999b7e Mon Sep 17 00:00:00 2001
From: Nick Peihl <nickpeihl@gmail.com>
Date: Fri, 20 Jul 2018 11:23:29 -0700
Subject: [PATCH 123/260] Add region ISO code to GeoIP Ingest plugin (#31669)

---
 docs/plugins/ingest-geoip.asciidoc             |  4 ++--
 .../index/mapper/filebeat-6.0.template.json    |  8 ++++----
 .../ingest/geoip/GeoIpProcessor.java           | 18 +++++++++++++++---
 .../geoip/GeoIpProcessorFactoryTests.java      |  2 +-
 .../ingest/geoip/GeoIpProcessorTests.java      |  3 ++-
 .../test/ingest_geoip/20_geoip_processor.yml   | 11 +++++++----
 6 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc
index 32516d07bef..688b36042ea 100644
--- a/docs/plugins/ingest-geoip.asciidoc
+++ b/docs/plugins/ingest-geoip.asciidoc
@@ -26,14 +26,14 @@ include::install_remove.asciidoc[]
 | `field`                | yes       | -                                                                                  | The field to get the ip address from for the geographical lookup.
 | `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
 | `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb, GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb files.
-| `properties`           | no        | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
+| `properties`           | no        | [`continent_name`, `country_iso_code`, `region_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 | `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 *Depends on what is available in `database_field`:
 
 * If the GeoLite2 City database is used, then the following fields may be added under the `target_field`: `ip`,
-`country_iso_code`, `country_name`, `continent_name`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude`
+`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude`
 and `location`. The fields actually added depend on what has been found and which properties were configured in `properties`.
 * If the GeoLite2 Country database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties
diff --git a/modules/mapper-extras/src/test/resources/org/elasticsearch/index/mapper/filebeat-6.0.template.json b/modules/mapper-extras/src/test/resources/org/elasticsearch/index/mapper/filebeat-6.0.template.json
index 0129509b5c0..fbaaf92351c 100644
--- a/modules/mapper-extras/src/test/resources/org/elasticsearch/index/mapper/filebeat-6.0.template.json
+++ b/modules/mapper-extras/src/test/resources/org/elasticsearch/index/mapper/filebeat-6.0.template.json
@@ -1,7 +1,5 @@
 {
-  "index_patterns": [
-    "filebeat-6.0.0-*"
-  ],
+  "index_patterns": ["filebeat-6.0.0-*"],
   "mappings": {
     "doc": {
       "_meta": {
@@ -67,12 +65,14 @@
                       "type": "keyword"
                     },
                     "country_iso_code": {
-                      "ignore_above": 1024,
                       "type": "keyword"
                     },
                     "location": {
                       "type": "geo_point"
                     },
+                    "region_iso_code": {
+                      "type": "keyword"
+                    },
                     "region_name": {
                       "ignore_above": 1024,
                       "type": "keyword"
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index f1b4b33017e..366b6ffc1d2 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -185,6 +185,16 @@ public final class GeoIpProcessor extends AbstractProcessor {
                         geoData.put("continent_name", continentName);
                     }
                     break;
+                case REGION_ISO_CODE:
+                    // ISO 3166-2 code for country subdivisions.
+                    // See iso.org/iso-3166-country-codes.html
+                    String countryIso = country.getIsoCode();
+                    String subdivisionIso = subdivision.getIsoCode();
+                    if (countryIso != null && subdivisionIso != null) {
+                        String regionIsoCode = countryIso + "-" + subdivisionIso;
+                        geoData.put("region_iso_code", regionIsoCode);
+                    }
+                    break;
                 case REGION_NAME:
                     String subdivisionName = subdivision.getName();
                     if (subdivisionName != null) {
@@ -300,8 +310,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
 
     public static final class Factory implements Processor.Factory {
         static final Set<Property> DEFAULT_CITY_PROPERTIES = EnumSet.of(
-            Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME,
-            Property.CITY_NAME, Property.LOCATION
+            Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_ISO_CODE,
+            Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION
         );
         static final Set<Property> DEFAULT_COUNTRY_PROPERTIES = EnumSet.of(
             Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE
@@ -377,6 +387,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
         COUNTRY_ISO_CODE,
         COUNTRY_NAME,
         CONTINENT_NAME,
+        REGION_ISO_CODE,
         REGION_NAME,
         CITY_NAME,
         TIMEZONE,
@@ -386,7 +397,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
 
         static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.of(
             Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME,
-            Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE, Property.LOCATION
+            Property.REGION_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE,
+            Property.LOCATION
         );
         static final EnumSet<Property> ALL_COUNTRY_PROPERTIES = EnumSet.of(
             Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 0aa2eb9fdfa..7a5d6f5808f 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -284,7 +284,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config1.put("properties", Collections.singletonList("invalid"));
         Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config1));
         assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " +
-            "COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"));
+            "COUNTRY_NAME, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"));
 
         Map<String, Object> config2 = new HashMap<>();
         config2.put("field", "_field");
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
index 48a1769cbf8..4c04d4e340a 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -117,11 +117,12 @@ public class GeoIpProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(address));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData.size(), equalTo(8));
+        assertThat(geoData.size(), equalTo(9));
         assertThat(geoData.get("ip"), equalTo(address));
         assertThat(geoData.get("country_iso_code"), equalTo("US"));
         assertThat(geoData.get("country_name"), equalTo("United States"));
         assertThat(geoData.get("continent_name"), equalTo("North America"));
+        assertThat(geoData.get("region_iso_code"), equalTo("US-FL"));
         assertThat(geoData.get("region_name"), equalTo("Florida"));
         assertThat(geoData.get("city_name"), equalTo("Hollywood"));
         assertThat(geoData.get("timezone"), equalTo("America/New_York"));
diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml
index 0c400c3c0ea..012ca717318 100644
--- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml
+++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml
@@ -30,11 +30,12 @@
         type: test
         id: 1
   - match: { _source.field1: "128.101.101.101" }
-  - length: { _source.geoip: 5 }
+  - length: { _source.geoip: 6 }
   - match: { _source.geoip.city_name: "Minneapolis" }
   - match: { _source.geoip.country_iso_code: "US" }
   - match: { _source.geoip.location.lon: -93.2166 }
   - match: { _source.geoip.location.lat: 44.9759 }
+  - match: { _source.geoip.region_iso_code: "US-MN" }
   - match: { _source.geoip.region_name: "Minnesota" }
   - match: { _source.geoip.continent_name: "North America" }
 
@@ -54,7 +55,7 @@
               {
                 "geoip" : {
                   "field" : "field1",
-                  "properties" : ["city_name", "country_iso_code", "ip", "location", "timezone", "country_name", "region_name", "continent_name"]
+                  "properties" : ["city_name", "country_iso_code", "ip", "location", "timezone", "country_name", "region_iso_code", "region_name", "continent_name"]
                 }
               }
             ]
@@ -75,7 +76,7 @@
         type: test
         id: 1
   - match: { _source.field1: "128.101.101.101" }
-  - length: { _source.geoip: 8 }
+  - length: { _source.geoip: 9 }
   - match: { _source.geoip.city_name: "Minneapolis" }
   - match: { _source.geoip.country_iso_code: "US" }
   - match: { _source.geoip.ip: "128.101.101.101" }
@@ -83,6 +84,7 @@
   - match: { _source.geoip.location.lat: 44.9759 }
   - match: { _source.geoip.timezone: "America/Chicago" }
   - match: { _source.geoip.country_name: "United States" }
+  - match: { _source.geoip.region_iso_code: "US-MN" }
   - match: { _source.geoip.region_name: "Minnesota" }
   - match: { _source.geoip.continent_name: "North America" }
 
@@ -188,11 +190,12 @@
         type: test
         id: 2
   - match: { _source.field1: "128.101.101.101" }
-  - length: { _source.geoip: 5 }
+  - length: { _source.geoip: 6 }
   - match: { _source.geoip.city_name: "Minneapolis" }
   - match: { _source.geoip.country_iso_code: "US" }
   - match: { _source.geoip.location.lon: -93.2166 }
   - match: { _source.geoip.location.lat: 44.9759 }
+  - match: { _source.geoip.region_iso_code: "US-MN" }
   - match: { _source.geoip.region_name: "Minnesota" }
   - match: { _source.geoip.continent_name: "North America" }
 

From e21692e3870a9e4437f9884628bfb90ca19fe3dd Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 20 Jul 2018 21:10:35 +0200
Subject: [PATCH 124/260] INGEST: Make a few Processors callable by Painless
 (#32170)

* INGEST: Make a few Processors callable by Painless
* Extracted a few stateless String processors as well as the json processor to static methods and whitelisted them in Painless
* provide whitelist from processors plugin
---
 modules/ingest-common/build.gradle            |   6 +
 .../ingest/common/BytesProcessor.java         |   6 +-
 .../ingest/common/JsonProcessor.java          |  38 +--
 .../ingest/common/LowercaseProcessor.java     |   6 +-
 .../ingest/common/Processors.java             |  49 ++++
 .../common/ProcessorsWhitelistExtension.java  |  41 ++++
 .../ingest/common/URLDecodeProcessor.java     |  10 +-
 .../ingest/common/UppercaseProcessor.java     |   6 +-
 ...asticsearch.painless.spi.PainlessExtension |   1 +
 .../ingest/common/processors_whitelist.txt    |  29 +++
 .../ingest/common/BytesProcessorTests.java    |   4 +-
 .../test/ingest/190_script_processor.yml      | 216 ++++++++++++++++++
 .../painless/spi/org.elasticsearch.txt        |   2 +-
 13 files changed, 391 insertions(+), 23 deletions(-)
 create mode 100644 modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java
 create mode 100644 modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java
 create mode 100644 modules/ingest-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
 create mode 100644 modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt
 create mode 100644 modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml

diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
index 424c1197da3..4f35bbee28d 100644
--- a/modules/ingest-common/build.gradle
+++ b/modules/ingest-common/build.gradle
@@ -20,11 +20,17 @@
 esplugin {
     description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
     classname 'org.elasticsearch.ingest.common.IngestCommonPlugin'
+    extendedPlugins = ['lang-painless']
 }
 
 dependencies {
+    compileOnly project(':modules:lang-painless')
     compile project(':libs:grok')
 }
 
 compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
 compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
+
+integTestCluster {
+    module project(':modules:lang-painless')
+}
\ No newline at end of file
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java
index dfe9a054acf..d07b56e1b3d 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java
@@ -35,9 +35,13 @@ public final class BytesProcessor extends AbstractStringProcessor {
         super(processorTag, field, ignoreMissing, targetField);
     }
 
+    public static long apply(String value) {
+        return ByteSizeValue.parseBytesSizeValue(value, null, "Ingest Field").getBytes();
+    }
+
     @Override
     protected Long process(String value) {
-        return ByteSizeValue.parseBytesSizeValue(value, null, getField()).getBytes();
+        return apply(value);
     }
 
     @Override
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
index 2f217735df2..c0a9d37abda 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
@@ -67,13 +67,11 @@ public final class JsonProcessor extends AbstractProcessor {
         return addToRoot;
     }
 
-    @Override
-    public void execute(IngestDocument document) throws Exception {
-        Object fieldValue = document.getFieldValue(field, Object.class);
-        BytesReference bytesRef = (fieldValue == null) ? new BytesArray("null") : new BytesArray(fieldValue.toString());
+    public static Object apply(Object fieldValue) {
+        BytesReference bytesRef = fieldValue == null ? new BytesArray("null") : new BytesArray(fieldValue.toString());
         try (InputStream stream = bytesRef.streamInput();
              XContentParser parser = JsonXContent.jsonXContent
-                .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
+                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
             XContentParser.Token token = parser.nextToken();
             Object value = null;
             if (token == XContentParser.Token.VALUE_NULL) {
@@ -91,20 +89,32 @@ public final class JsonProcessor extends AbstractProcessor {
             } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
                 throw new IllegalArgumentException("cannot read binary value");
             }
-            if (addToRoot && (value instanceof Map)) {
-                for (Map.Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
-                    document.setFieldValue(entry.getKey(), entry.getValue());
-                }
-            } else if (addToRoot) {
-                throw new IllegalArgumentException("cannot add non-map fields to root of document");
-            } else {
-                document.setFieldValue(targetField, value);
-            }
+            return value;
         } catch (IOException e) {
             throw new IllegalArgumentException(e);
         }
     }
 
+    public static void apply(Map<String, Object> ctx, String fieldName) {
+        Object value = apply(ctx.get(fieldName));
+        if (value instanceof Map) {
+            @SuppressWarnings("unchecked")
+                Map<String, Object> map = (Map<String, Object>) value;
+                ctx.putAll(map);
+        } else {
+            throw new IllegalArgumentException("cannot add non-map fields to root of document");
+        }
+    }
+
+    @Override
+    public void execute(IngestDocument document) throws Exception {
+        if (addToRoot) {
+           apply(document.getSourceAndMetadata(), field);
+        } else {
+            document.setFieldValue(targetField, apply(document.getFieldValue(field, Object.class)));
+        }
+    }
+
     @Override
     public String getType() {
         return TYPE;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
index aef8b0cce24..4269cb05257 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
@@ -35,9 +35,13 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
         super(processorTag, field, ignoreMissing, targetField);
     }
 
+    public static String apply(String value) {
+        return value.toLowerCase(Locale.ROOT);
+    }
+
     @Override
     protected String process(String value) {
-        return value.toLowerCase(Locale.ROOT);
+        return apply(value);
     }
 
     @Override
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java
new file mode 100644
index 00000000000..8a0b1529892
--- /dev/null
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import java.util.Map;
+
+public final class Processors {
+
+    public static long bytes(String value) {
+        return BytesProcessor.apply(value);
+    }
+
+    public static String lowercase(String value) {
+        return LowercaseProcessor.apply(value);
+    }
+
+    public static String uppercase(String value) {
+        return UppercaseProcessor.apply(value);
+    }
+
+    public static Object json(Object fieldValue) {
+        return JsonProcessor.apply(fieldValue);
+    }
+
+    public static void json(Map<String, Object> ctx, String field) {
+        JsonProcessor.apply(ctx, field);
+    }
+
+    public static String urlDecode(String value) {
+        return URLDecodeProcessor.apply(value);
+    }
+}
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java
new file mode 100644
index 00000000000..ced84057c7a
--- /dev/null
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ProcessorsWhitelistExtension.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.painless.spi.PainlessExtension;
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.script.IngestScript;
+import org.elasticsearch.script.ScriptContext;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class ProcessorsWhitelistExtension implements PainlessExtension {
+
+    private static final Whitelist WHITELIST =
+        WhitelistLoader.loadFromResourceFiles(ProcessorsWhitelistExtension.class, "processors_whitelist.txt");
+
+    @Override
+    public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
+        return Collections.singletonMap(IngestScript.CONTEXT, Collections.singletonList(WHITELIST));
+    }
+}
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java
index 945419499ad..fb6c5acf98b 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/URLDecodeProcessor.java
@@ -34,15 +34,19 @@ public final class URLDecodeProcessor extends AbstractStringProcessor {
         super(processorTag, field, ignoreMissing, targetField);
     }
 
-    @Override
-    protected String process(String value) {
+    public static String apply(String value) {
         try {
             return URLDecoder.decode(value, "UTF-8");
         } catch (UnsupportedEncodingException e) {
-            throw new IllegalArgumentException("could not URL-decode field[" + getField() + "]", e);
+            throw new IllegalArgumentException("Could not URL-decode value.", e);
         }
     }
 
+    @Override
+    protected String process(String value) {
+        return apply(value);
+    }
+
     @Override
     public String getType() {
         return TYPE;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
index af93f06a8f2..6c428627c7d 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
@@ -34,9 +34,13 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
         super(processorTag, field, ignoreMissing, targetField);
     }
 
+    public static String apply(String value) {
+        return value.toUpperCase(Locale.ROOT);
+    }
+
     @Override
     protected String process(String value) {
-        return value.toUpperCase(Locale.ROOT);
+        return apply(value);
     }
 
     @Override
diff --git a/modules/ingest-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/modules/ingest-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
new file mode 100644
index 00000000000..8a98f034be5
--- /dev/null
+++ b/modules/ingest-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
@@ -0,0 +1 @@
+org.elasticsearch.ingest.common.ProcessorsWhitelistExtension
\ No newline at end of file
diff --git a/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt
new file mode 100644
index 00000000000..3d93b19f066
--- /dev/null
+++ b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt
@@ -0,0 +1,29 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This file contains a whitelist of static processor methods that can be accessed from painless
+
+class org.elasticsearch.ingest.common.Processors {
+  long bytes(String)
+  String lowercase(String)
+  String uppercase(String)
+  Object json(Object)
+  void json(Map, String)
+  String urlDecode(String)
+}
\ No newline at end of file
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java
index 0da3434adf1..788340a455a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java
@@ -63,7 +63,7 @@ public class BytesProcessorTests extends AbstractStringProcessorTestCase {
         Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument));
         assertThat(exception.getMessage(),
-            CoreMatchers.equalTo("failed to parse setting [" + fieldName + "] with value [8912pb] as a size in bytes"));
+            CoreMatchers.equalTo("failed to parse setting [Ingest Field] with value [8912pb] as a size in bytes"));
         assertThat(exception.getCause().getMessage(),
             CoreMatchers.containsString("Values greater than 9223372036854775807 bytes are not supported"));
     }
@@ -93,6 +93,6 @@ public class BytesProcessorTests extends AbstractStringProcessorTestCase {
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, expectedResultType()), equalTo(1126L));
         assertWarnings("Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for setting " +
-            "[" + fieldName + "]");
+            "[Ingest Field]");
     }
 }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml
new file mode 100644
index 00000000000..bd55b764a95
--- /dev/null
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml
@@ -0,0 +1,216 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "my_pipeline"
+        ignore: 404
+
+---
+"Test invoke bytes processor":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "ctx.target_field = Processors.bytes(ctx.source_field)"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {source_field: "1kb"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.source_field: "1kb" }
+  - match: { _source.target_field: 1024 }
+
+---
+"Test invoke lowercase processor":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "ctx.target_field = Processors.lowercase(ctx.source_field)"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {source_field: "FooBar"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.source_field: "FooBar" }
+  - match: { _source.target_field: "foobar" }
+
+---
+"Test invoke uppercase processor":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "ctx.target_field = Processors.uppercase(ctx.source_field)"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {source_field: "FooBar"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.source_field: "FooBar" }
+  - match: { _source.target_field: "FOOBAR" }
+
+---
+"Test invoke json processor, assign to field":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "ctx.target_field = Processors.json(ctx.source_field)"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {source_field: "{\"foo\":\"bar\"}"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.source_field: "{\"foo\":\"bar\"}" }
+  - match: { _source.target_field.foo: "bar" }
+
+---
+"Test invoke json processor, assign to root":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "Processors.json(ctx, 'source_field')"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {source_field: "{\"foo\":\"bar\"}"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.source_field: "{\"foo\":\"bar\"}" }
+  - match: { _source.foo: "bar" }
+
+---
+"Test invoke urlDecode processor":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "ctx.target_field = Processors.urlDecode(ctx.source_field)"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {source_field: "foo%20bar"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.source_field: "foo%20bar" }
+  - match: { _source.target_field: "foo bar" }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
index 6495659d9cd..8491d15c27e 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
@@ -174,4 +174,4 @@ class org.elasticsearch.index.similarity.ScriptedSimilarity$Term {
 class org.elasticsearch.index.similarity.ScriptedSimilarity$Doc {
   int getLength()
   float getFreq()
-}
+}
\ No newline at end of file

From 7aa8a0a92787b116edd9044c131115e68459d932 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 20 Jul 2018 22:32:50 +0200
Subject: [PATCH 125/260] INGEST: Extend KV Processor (#31789) (#32232)

* INGEST: Extend KV Processor (#31789)

Added more capabilities supported by LS to the KV processor:
* Stripping of brackets and quotes from values (`include_brackets` in corresponding LS filter)
* Adding key prefixes
* Trimming specified chars from keys and values

Refactored the way the filter is configured to avoid conditionals during execution.
Refactored Tests a little to not have to add more redundant getters for new parameters.

Relates #31786
* Add documentation
---
 docs/reference/ingest/ingest-node.asciidoc    |   4 +
 .../ingest/common/KeyValueProcessor.java      | 131 ++++++++++++++----
 .../ingest/common/KeyValueProcessorTests.java | 116 ++++++++++++++--
 3 files changed, 214 insertions(+), 37 deletions(-)

diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 3991b738e13..79277d22e81 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -1732,6 +1732,10 @@ For example, if you have a log message which contains `ip=1.2.3.4 error=REFUSED`
 | `include_keys`   | no        | `null`   | List of keys to filter and insert into document. Defaults to including all keys
 | `exclude_keys`   | no        | `null`   | List of keys to exclude from document
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+| `prefix`         | no        | `null`   | Prefix to be added to extracted keys
+| `trim_key`       | no        | `null`   | String of characters to trim from extracted keys
+| `trim_value`     | no        | `null`   | String of characters to trim from extracted values
+| `strip_brackets` | no        | `false`  | If `true` strip brackets `()`, `<>`, `[]` as well as quotes `'` and `"` from extracted values
 |======
 
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java
index 6ed065926d6..9cce3cedf3d 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java
@@ -25,11 +25,14 @@ import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
 
 /**
  * The KeyValueProcessor parses and extracts messages of the `key=value` variety into fields with values of the keys.
@@ -38,6 +41,8 @@ public final class KeyValueProcessor extends AbstractProcessor {
 
     public static final String TYPE = "kv";
 
+    private static final Pattern STRIP_BRACKETS = Pattern.compile("(^[\\(\\[<\"'])|([\\]\\)>\"']$)");
+
     private final String field;
     private final String fieldSplit;
     private final String valueSplit;
@@ -45,9 +50,11 @@ public final class KeyValueProcessor extends AbstractProcessor {
     private final Set<String> excludeKeys;
     private final String targetField;
     private final boolean ignoreMissing;
+    private final Consumer<IngestDocument> execution;
 
     KeyValueProcessor(String tag, String field, String fieldSplit, String valueSplit, Set<String> includeKeys,
-                      Set<String> excludeKeys, String targetField, boolean ignoreMissing) {
+                      Set<String> excludeKeys, String targetField, boolean ignoreMissing,
+                      String trimKey, String trimValue, boolean stripBrackets, String prefix) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
@@ -56,6 +63,92 @@ public final class KeyValueProcessor extends AbstractProcessor {
         this.includeKeys = includeKeys;
         this.excludeKeys = excludeKeys;
         this.ignoreMissing = ignoreMissing;
+        this.execution = buildExecution(
+            fieldSplit, valueSplit, field, includeKeys, excludeKeys, targetField, ignoreMissing, trimKey, trimValue,
+            stripBrackets, prefix
+        );
+    }
+
+    private static Consumer<IngestDocument> buildExecution(String fieldSplit, String valueSplit, String field,
+                                                           Set<String> includeKeys, Set<String> excludeKeys,
+                                                           String targetField, boolean ignoreMissing,
+                                                           String trimKey, String trimValue, boolean stripBrackets,
+                                                           String prefix) {
+        final Predicate<String> keyFilter;
+        if (includeKeys == null) {
+            if (excludeKeys == null) {
+                keyFilter = key -> true;
+            } else {
+                keyFilter = key -> excludeKeys.contains(key) == false;
+            }
+        } else {
+            if (excludeKeys == null) {
+                keyFilter = includeKeys::contains;
+            } else {
+                keyFilter = key -> includeKeys.contains(key) && excludeKeys.contains(key) == false;
+            }
+        }
+        final String fieldPathPrefix;
+        String keyPrefix = prefix == null ? "" : prefix;
+        if (targetField == null) {
+            fieldPathPrefix = keyPrefix;
+        } else {
+            fieldPathPrefix = targetField + "." + keyPrefix;
+        }
+        final Function<String, String> keyPrefixer;
+        if (fieldPathPrefix.isEmpty()) {
+            keyPrefixer = val -> val;
+        } else {
+            keyPrefixer = val -> fieldPathPrefix + val;
+        }
+        final Function<String, String[]> fieldSplitter = buildSplitter(fieldSplit, true);
+        Function<String, String[]> valueSplitter = buildSplitter(valueSplit, false);
+        final Function<String, String> keyTrimmer = buildTrimmer(trimKey);
+        final Function<String, String> bracketStrip;
+        if (stripBrackets) {
+            bracketStrip = val -> STRIP_BRACKETS.matcher(val).replaceAll("");
+        } else {
+            bracketStrip = val -> val;
+        }
+        final Function<String, String> valueTrimmer = buildTrimmer(trimValue);
+        return document -> {
+            String value = document.getFieldValue(field, String.class, ignoreMissing);
+            if (value == null) {
+                if (ignoreMissing) {
+                    return;
+                }
+                throw new IllegalArgumentException("field [" + field + "] is null, cannot extract key-value pairs.");
+            }
+            for (String part : fieldSplitter.apply(value)) {
+                String[] kv = valueSplitter.apply(part);
+                if (kv.length != 2) {
+                    throw new IllegalArgumentException("field [" + field + "] does not contain value_split [" + valueSplit + "]");
+                }
+                String key = keyTrimmer.apply(kv[0]);
+                if (keyFilter.test(key)) {
+                    append(document, keyPrefixer.apply(key), valueTrimmer.apply(bracketStrip.apply(kv[1])));
+                }
+            }
+        };
+    }
+
+    private static Function<String, String> buildTrimmer(String trim) {
+        if (trim == null) {
+            return val -> val;
+        } else {
+            Pattern pattern = Pattern.compile("(^([" + trim + "]+))|([" + trim + "]+$)");
+            return val -> pattern.matcher(val).replaceAll("");
+        }
+    }
+
+    private static Function<String, String[]> buildSplitter(String split, boolean fields) {
+        int limit = fields ? 0 : 2;
+        if (split.length() > 2 || split.length() == 2 && split.charAt(0) != '\\') {
+            Pattern splitPattern = Pattern.compile(split);
+            return val -> splitPattern.split(val, limit);
+        } else {
+            return val -> val.split(split, limit);
+        }
     }
 
     String getField() {
@@ -86,7 +179,7 @@ public final class KeyValueProcessor extends AbstractProcessor {
         return ignoreMissing;
     }
 
-    public void append(IngestDocument document, String targetField, String value) {
+    private static void append(IngestDocument document, String targetField, String value) {
         if (document.hasField(targetField)) {
             document.appendFieldValue(targetField, value);
         } else {
@@ -96,27 +189,7 @@ public final class KeyValueProcessor extends AbstractProcessor {
 
     @Override
     public void execute(IngestDocument document) {
-        String oldVal = document.getFieldValue(field, String.class, ignoreMissing);
-
-        if (oldVal == null && ignoreMissing) {
-            return;
-        } else if (oldVal == null) {
-            throw new IllegalArgumentException("field [" + field + "] is null, cannot extract key-value pairs.");
-        }
-
-        String fieldPathPrefix = (targetField == null) ? "" : targetField + ".";
-        Arrays.stream(oldVal.split(fieldSplit))
-            .map((f) -> {
-                String[] kv = f.split(valueSplit, 2);
-                if (kv.length != 2) {
-                    throw new IllegalArgumentException("field [" + field + "] does not contain value_split [" + valueSplit + "]");
-                }
-                return kv;
-            })
-            .filter((p) ->
-                (includeKeys == null || includeKeys.contains(p[0])) &&
-                    (excludeKeys == null || excludeKeys.contains(p[0]) == false))
-            .forEach((p) -> append(document, fieldPathPrefix + p[0], p[1]));
+        execution.accept(document);
     }
 
     @Override
@@ -132,6 +205,11 @@ public final class KeyValueProcessor extends AbstractProcessor {
             String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field");
             String fieldSplit = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field_split");
             String valueSplit = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "value_split");
+            String trimKey = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "trim_key");
+            String trimValue = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "trim_value");
+            String prefix = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "prefix");
+            boolean stripBrackets =
+                ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "strip_brackets", false);
             Set<String> includeKeys = null;
             Set<String> excludeKeys = null;
             List<String> includeKeysList = ConfigurationUtils.readOptionalList(TYPE, processorTag, config, "include_keys");
@@ -143,7 +221,10 @@ public final class KeyValueProcessor extends AbstractProcessor {
                 excludeKeys = Collections.unmodifiableSet(Sets.newHashSet(excludeKeysList));
             }
             boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
-            return new KeyValueProcessor(processorTag, field, fieldSplit, valueSplit, includeKeys, excludeKeys, targetField, ignoreMissing);
+            return new KeyValueProcessor(
+                processorTag, field, fieldSplit, valueSplit, includeKeys, excludeKeys, targetField, ignoreMissing,
+                trimKey, trimValue, stripBrackets, prefix
+            );
         }
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
index 380af44c251..591f9994c60 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
@@ -25,19 +25,25 @@ import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 
 public class KeyValueProcessorTests extends ESTestCase {
 
+    private static final KeyValueProcessor.Factory FACTORY = new KeyValueProcessor.Factory();
+
     public void test() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=", null, null, "target", false);
+        Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
         assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(Arrays.asList("world", "universe")));
@@ -46,7 +52,7 @@ public class KeyValueProcessorTests extends ESTestCase {
     public void testRootTarget() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         ingestDocument.setFieldValue("myField", "first=hello&second=world&second=universe");
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "myField", "&", "=", null, null,null, false);
+        Processor processor = createKvProcessor("myField", "&", "=", null, null,null, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("first", String.class), equalTo("hello"));
         assertThat(ingestDocument.getFieldValue("second", List.class), equalTo(Arrays.asList("world", "universe")));
@@ -55,7 +61,7 @@ public class KeyValueProcessorTests extends ESTestCase {
     public void testKeySameAsSourceField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         ingestDocument.setFieldValue("first", "first=hello");
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "first", "&", "=", null, null,null, false);
+        Processor processor = createKvProcessor("first", "&", "=", null, null,null, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("first", List.class), equalTo(Arrays.asList("first=hello", "hello")));
     }
@@ -63,7 +69,7 @@ public class KeyValueProcessorTests extends ESTestCase {
     public void testIncludeKeys() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=",
+        Processor processor = createKvProcessor(fieldName, "&", "=",
             Sets.newHashSet("first"), null, "target", false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
@@ -73,7 +79,7 @@ public class KeyValueProcessorTests extends ESTestCase {
     public void testExcludeKeys() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=",
+        Processor processor = createKvProcessor(fieldName, "&", "=",
             null, Sets.newHashSet("second"), "target", false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
@@ -84,7 +90,7 @@ public class KeyValueProcessorTests extends ESTestCase {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument,
             "first=hello&second=world&second=universe&third=bar");
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=",
+        Processor processor = createKvProcessor(fieldName, "&", "=",
             Sets.newHashSet("first", "second"), Sets.newHashSet("first", "second"), "target", false);
         processor.execute(ingestDocument);
         assertFalse(ingestDocument.hasField("target.first"));
@@ -92,9 +98,9 @@ public class KeyValueProcessorTests extends ESTestCase {
         assertFalse(ingestDocument.hasField("target.third"));
     }
 
-    public void testMissingField() {
+    public void testMissingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "unknown", "&",
+        Processor processor = createKvProcessor("unknown", "&",
             "=", null, null, "target", false);
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
         assertThat(exception.getMessage(), equalTo("field [unknown] not present as part of path [unknown]"));
@@ -105,7 +111,7 @@ public class KeyValueProcessorTests extends ESTestCase {
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap(fieldName, null));
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "", "", null, null, "target", true);
+        Processor processor = createKvProcessor(fieldName, "", "", null, null, "target", true);
         processor.execute(ingestDocument);
         assertIngestDocument(originalIngestDocument, ingestDocument);
     }
@@ -113,7 +119,7 @@ public class KeyValueProcessorTests extends ESTestCase {
     public void testNonExistentWithIgnoreMissing() throws Exception {
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "unknown", "", "", null, null, "target", true);
+        Processor processor = createKvProcessor("unknown", "", "", null, null, "target", true);
         processor.execute(ingestDocument);
         assertIngestDocument(originalIngestDocument, ingestDocument);
     }
@@ -121,7 +127,7 @@ public class KeyValueProcessorTests extends ESTestCase {
     public void testFailFieldSplitMatch() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello|second=world|second=universe");
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), fieldName, "&", "=", null, null, "target", false);
+        Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello|second=world|second=universe"));
         assertFalse(ingestDocument.hasField("target.second"));
@@ -129,8 +135,94 @@ public class KeyValueProcessorTests extends ESTestCase {
 
     public void testFailValueSplitMatch() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("foo", "bar"));
-        Processor processor = new KeyValueProcessor(randomAlphaOfLength(10), "foo", "&", "=", null, null, "target", false);
+        Processor processor = createKvProcessor("foo", "&", "=", null, null, "target", false);
         Exception exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
         assertThat(exception.getMessage(), equalTo("field [foo] does not contain value_split [=]"));
     }
+
+    public void testTrimKeyAndValue() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first= hello &second=world& second =universe");
+        Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false, " ", " ", false, null);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
+        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(Arrays.asList("world", "universe")));
+    }
+
+    public void testTrimMultiCharSequence() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument,
+            "to=<foo@example.com>, orig_to=<bar@example.com>, %+relay=mail.example.com[private/dovecot-lmtp]," +
+                " delay=2.2, delays=1.9/0.01/0.01/0.21, dsn=2.0.0, status=sent "
+        );
+        Processor processor = createKvProcessor(fieldName, " ", "=", null, null, "target", false, "%+", "<>,", false, null);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.to", String.class), equalTo("foo@example.com"));
+        assertThat(ingestDocument.getFieldValue("target.orig_to", String.class), equalTo("bar@example.com"));
+        assertThat(ingestDocument.getFieldValue("target.relay", String.class), equalTo("mail.example.com[private/dovecot-lmtp]"));
+        assertThat(ingestDocument.getFieldValue("target.delay", String.class), equalTo("2.2"));
+        assertThat(ingestDocument.getFieldValue("target.delays", String.class), equalTo("1.9/0.01/0.01/0.21"));
+        assertThat(ingestDocument.getFieldValue("target.dsn", String.class), equalTo("2.0.0"));
+        assertThat(ingestDocument.getFieldValue("target.status", String.class), equalTo("sent"));
+    }
+
+    public void testStripBrackets() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(
+            random(), ingestDocument, "first=<hello>&second=\"world\"&second=(universe)&third=<foo>&fourth=[bar]&fifth='last'"
+        );
+        Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false, null, null, true, null);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
+        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(Arrays.asList("world", "universe")));
+        assertThat(ingestDocument.getFieldValue("target.third", String.class), equalTo("foo"));
+        assertThat(ingestDocument.getFieldValue("target.fourth", String.class), equalTo("bar"));
+        assertThat(ingestDocument.getFieldValue("target.fifth", String.class), equalTo("last"));
+    }
+
+    public void testAddPrefix() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
+        Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false, null, null, false, "arg_");
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.arg_first", String.class), equalTo("hello"));
+        assertThat(ingestDocument.getFieldValue("target.arg_second", List.class), equalTo(Arrays.asList("world", "universe")));
+    }
+
+    private static KeyValueProcessor createKvProcessor(String field, String fieldSplit, String valueSplit, Set<String> includeKeys,
+                                                       Set<String> excludeKeys, String targetField,
+                                                       boolean ignoreMissing) throws Exception {
+        return createKvProcessor(
+            field, fieldSplit, valueSplit, includeKeys, excludeKeys, targetField, ignoreMissing, null, null, false, null
+        );
+    }
+
+    private static KeyValueProcessor createKvProcessor(String field, String fieldSplit, String valueSplit, Set<String> includeKeys,
+                                                       Set<String> excludeKeys, String targetField, boolean ignoreMissing,
+                                                       String trimKey, String trimValue, boolean stripBrackets,
+                                                       String prefix) throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", field);
+        config.put("field_split", fieldSplit);
+        config.put("value_split", valueSplit);
+        config.put("target_field", targetField);
+        if (includeKeys != null) {
+            config.put("include_keys", new ArrayList<>(includeKeys));
+        }
+        if (excludeKeys != null) {
+            config.put("exclude_keys", new ArrayList<>(excludeKeys));
+        }
+        config.put("ignore_missing", ignoreMissing);
+        if (trimKey != null) {
+            config.put("trim_key", trimKey);
+        }
+        if (trimValue != null) {
+            config.put("trim_value", trimValue);
+        }
+        config.put("strip_brackets", stripBrackets);
+        if (prefix != null) {
+            config.put("prefix", prefix);
+        }
+        return FACTORY.create(null, randomAlphaOfLength(10), config);
+    }
 }

From aaa8f842d60f45d6303fb04a9f4fca4ca40dc4ca Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ikakavas@protonmail.com>
Date: Sat, 21 Jul 2018 00:03:58 +0300
Subject: [PATCH 126/260] Remove BouncyCastle dependency from runtime (#32193)

* Remove BouncyCastle dependency from runtime

This commit introduces a new gradle  project that contains
 the classes that have a dependency on BouncyCastle. For
the default distribution, It builds  a jar from those and
 in puts it in a subdirectory of lib
 (/tools/security-cli) along with the BouncyCastle jars.
This directory is then passed in the
ES_ADDITIONAL_CLASSPATH_DIRECTORIES of the CLI tools
that use these classes.

BouncyCastle is removed as a runtime dependency (remains
as a compileOnly one) from x-pack core and x-pack security.
---
 distribution/archives/build.gradle            |   2 +-
 distribution/build.gradle                     |  28 ++--
 distribution/packages/build.gradle            |   2 +-
 .../packaging/test/ArchiveTestCase.java       |  22 +++
 .../packaging/util/Installation.java          |   1 +
 x-pack/plugin/core/build.gradle               |   4 +-
 .../xpack/core/ssl/CertParsingUtils.java      |  10 +-
 x-pack/plugin/security/build.gradle           |   5 +-
 x-pack/plugin/security/cli/build.gradle       |  20 +++
 .../licenses/bcpkix-jdk15on-1.59.jar.sha1     |   0
 .../licenses/bcprov-jdk15on-1.59.jar.sha1     |   0
 .../cli}/licenses/bouncycastle-LICENSE.txt    |   0
 .../cli}/licenses/bouncycastle-NOTICE.txt     |   0
 .../xpack/security/cli}/CertGenUtils.java     |  22 +--
 .../cli}/CertificateGenerateTool.java         | 128 ++++++++++--------
 .../xpack/security/cli}/CertificateTool.java  | 120 ++++++++--------
 .../security/cli}/CertGenUtilsTests.java      |   3 +-
 .../cli}/CertificateGenerateToolTests.java    |  14 +-
 .../security/cli}/CertificateToolTests.java   |  20 +--
 .../xpack/security/cli/testnode.crt           |  23 ++++
 .../xpack/security/cli/testnode.pem           |  30 ++++
 .../src/main/bin/elasticsearch-certgen        |   3 +-
 .../src/main/bin/elasticsearch-certgen.bat    |   3 +-
 .../src/main/bin/elasticsearch-certutil       |   3 +-
 .../src/main/bin/elasticsearch-certutil.bat   |   3 +-
 25 files changed, 294 insertions(+), 172 deletions(-)
 create mode 100644 x-pack/plugin/security/cli/build.gradle
 rename x-pack/plugin/{core => security/cli}/licenses/bcpkix-jdk15on-1.59.jar.sha1 (100%)
 rename x-pack/plugin/{core => security/cli}/licenses/bcprov-jdk15on-1.59.jar.sha1 (100%)
 rename x-pack/plugin/{core => security/cli}/licenses/bouncycastle-LICENSE.txt (100%)
 rename x-pack/plugin/{core => security/cli}/licenses/bouncycastle-NOTICE.txt (100%)
 rename x-pack/plugin/{core/src/main/java/org/elasticsearch/xpack/core/ssl => security/cli/src/main/java/org/elasticsearch/xpack/security/cli}/CertGenUtils.java (93%)
 rename x-pack/plugin/{core/src/main/java/org/elasticsearch/xpack/core/ssl => security/cli/src/main/java/org/elasticsearch/xpack/security/cli}/CertificateGenerateTool.java (90%)
 rename x-pack/plugin/{core/src/main/java/org/elasticsearch/xpack/core/ssl => security/cli/src/main/java/org/elasticsearch/xpack/security/cli}/CertificateTool.java (92%)
 rename x-pack/plugin/{core/src/test/java/org/elasticsearch/xpack/core/ssl => security/cli/src/test/java/org/elasticsearch/xpack/security/cli}/CertGenUtilsTests.java (97%)
 rename x-pack/{qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl => plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli}/CertificateGenerateToolTests.java (98%)
 rename x-pack/{qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl => plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli}/CertificateToolTests.java (98%)
 create mode 100644 x-pack/plugin/security/cli/src/test/resources/org/elasticsearch/xpack/security/cli/testnode.crt
 create mode 100644 x-pack/plugin/security/cli/src/test/resources/org/elasticsearch/xpack/security/cli/testnode.pem

diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index 3c6780cba84..0269e4399ce 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -49,7 +49,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean os
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
-        with libFiles
+        with libFiles(oss)
       }
       into('config') {
         dirMode 0750
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 6ffb678cb2b..675799c5b22 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -227,16 +227,24 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     /*****************************************************************************
      *                   Common files in all distributions                       *
      *****************************************************************************/
-    libFiles = copySpec {
-      // delay by using closures, since they have not yet been configured, so no jar task exists yet
-      from { project(':server').jar }
-      from { project(':server').configurations.runtime }
-      from { project(':libs:plugin-classloader').jar }
-      from { project(':distribution:tools:java-version-checker').jar }
-      from { project(':distribution:tools:launchers').jar }
-      into('tools/plugin-cli') {
-        from { project(':distribution:tools:plugin-cli').jar }
-        from { project(':distribution:tools:plugin-cli').configurations.runtime }
+    libFiles = { oss ->
+      copySpec {
+        // delay by using closures, since they have not yet been configured, so no jar task exists yet
+        from { project(':server').jar }
+        from { project(':server').configurations.runtime }
+        from { project(':libs:plugin-classloader').jar }
+        from { project(':distribution:tools:java-version-checker').jar }
+        from { project(':distribution:tools:launchers').jar }
+        into('tools/plugin-cli') {
+          from { project(':distribution:tools:plugin-cli').jar }
+          from { project(':distribution:tools:plugin-cli').configurations.runtime }
+        }
+        if (oss == false) {
+          into('tools/security-cli') {
+            from { project(':x-pack:plugin:security:cli').jar }
+            from { project(':x-pack:plugin:security:cli').configurations.compile }
+          }
+        }
       }
     }
 
diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
index 04fa6313c0a..fcd69138da3 100644
--- a/distribution/packages/build.gradle
+++ b/distribution/packages/build.gradle
@@ -126,7 +126,7 @@ Closure commonPackageConfig(String type, boolean oss) {
       }
       into('lib') {
         with copySpec {
-          with libFiles
+          with libFiles(oss)
           // we need to specify every intermediate directory so we iterate through the parents; duplicate calls with the same part are fine
           eachFile { FileCopyDetails fcp ->
             String[] segments = fcp.relativePath.segments
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
index 3aada7837d8..20561115542 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
@@ -57,6 +57,7 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.isEmptyString;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
@@ -302,5 +303,26 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
         }
     }
 
+    public void test90SecurityCliPackaging() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Installation.Executables bin = installation.executables();
+        final Shell sh = new Shell();
+
+        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+            assertTrue(Files.exists(installation.lib.resolve("tools").resolve("security-cli")));
+            Platforms.onLinux(() -> {
+                final Result result = sh.run(bin.elasticsearchCertutil + " help");
+                assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack"));
+            });
+
+            Platforms.onWindows(() -> {
+                final Result result = sh.run(bin.elasticsearchCertutil + " help");
+                assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack"));
+            });
+        } else if (distribution().equals(Distribution.OSS_TAR) || distribution().equals(Distribution.OSS_ZIP)) {
+            assertFalse(Files.exists(installation.lib.resolve("tools").resolve("security-cli")));
+        }
+    }
 
 }
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java
index 40dc546f230..8bc3fc6e14d 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java
@@ -101,6 +101,7 @@ public class Installation {
         public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin");
         public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
         public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
+        public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
 
         private Path platformExecutable(String name) {
             final String platformExecutableName = Platforms.WINDOWS
diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index ca926fa0d54..7ed98ccb5b4 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -20,7 +20,6 @@ esplugin {
 }
 
 dependencyLicenses {
-  mapping from: /bc.*/, to: 'bouncycastle'
   mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
   mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
 }
@@ -38,8 +37,6 @@ dependencies {
 
     // security deps
     compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
-    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
     compile project(path: ':modules:transport-netty4', configuration: 'runtime')
 
     testCompile 'org.elasticsearch:securemock:1.2'
@@ -116,6 +113,7 @@ task testJar(type: Jar) {
     appendix 'test'
     from sourceSets.test.output
 }
+
 artifacts {
     // normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
     archives jar
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
index 1617a92b550..6503f686b64 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
@@ -63,7 +63,7 @@ public class CertParsingUtils {
         return PathUtils.get(path).normalize();
     }
 
-    static KeyStore readKeyStore(Path path, String type, char[] password)
+    public static KeyStore readKeyStore(Path path, String type, char[] password)
             throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException {
         try (InputStream in = Files.newInputStream(path)) {
             KeyStore store = KeyStore.getInstance(type);
@@ -108,7 +108,7 @@ public class CertParsingUtils {
         return certificates.toArray(new X509Certificate[0]);
     }
 
-    static List<Certificate> readCertificates(InputStream input) throws CertificateException, IOException {
+    public static List<Certificate> readCertificates(InputStream input) throws CertificateException, IOException {
         CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
         Collection<Certificate> certificates = (Collection<Certificate>) certFactory.generateCertificates(input);
         return new ArrayList<>(certificates);
@@ -140,7 +140,7 @@ public class CertParsingUtils {
     /**
      * Creates a {@link KeyStore} from a PEM encoded certificate and key file
      */
-    static KeyStore getKeyStoreFromPEM(Path certificatePath, Path keyPath, char[] keyPassword)
+    public static KeyStore getKeyStoreFromPEM(Path certificatePath, Path keyPath, char[] keyPassword)
             throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException {
         final PrivateKey key = PemUtils.readPrivateKey(keyPath, () -> keyPassword);
         final Certificate[] certificates = readCertificates(Collections.singletonList(certificatePath));
@@ -168,7 +168,7 @@ public class CertParsingUtils {
     /**
      * Returns a {@link X509ExtendedKeyManager} that is built from the provided keystore
      */
-    static X509ExtendedKeyManager keyManager(KeyStore keyStore, char[] password, String algorithm)
+    public static X509ExtendedKeyManager keyManager(KeyStore keyStore, char[] password, String algorithm)
             throws NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException {
         KeyManagerFactory kmf = KeyManagerFactory.getInstance(algorithm);
         kmf.init(keyStore, password);
@@ -271,7 +271,7 @@ public class CertParsingUtils {
     /**
      * Creates a {@link X509ExtendedTrustManager} based on the trust material in the provided {@link KeyStore}
      */
-    static X509ExtendedTrustManager trustManager(KeyStore keyStore, String algorithm)
+    public static X509ExtendedTrustManager trustManager(KeyStore keyStore, String algorithm)
             throws NoSuchAlgorithmException, KeyStoreException {
         TrustManagerFactory tmf = TrustManagerFactory.getInstance(algorithm);
         tmf.init(keyStore);
diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle
index 5bdfdf65014..854b0165954 100644
--- a/x-pack/plugin/security/build.gradle
+++ b/x-pack/plugin/security/build.gradle
@@ -22,8 +22,8 @@ dependencies {
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 
     compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
-    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
+    compileOnly 'org.bouncycastle:bcprov-jdk15on:1.59'
+    compileOnly 'org.bouncycastle:bcpkix-jdk15on:1.59'
 
     // the following are all SAML dependencies - might as well download the whole internet
     compile "org.opensaml:opensaml-core:3.3.0"
@@ -79,7 +79,6 @@ sourceSets.test.resources {
     srcDir '../core/src/test/resources'
 }
 dependencyLicenses {
-    mapping from: /bc.*/, to: 'bouncycastle'
     mapping from: /java-support|opensaml-.*/, to: 'shibboleth'
     mapping from: /http.*/, to: 'httpclient'
 }
diff --git a/x-pack/plugin/security/cli/build.gradle b/x-pack/plugin/security/cli/build.gradle
new file mode 100644
index 00000000000..1799a2c7b81
--- /dev/null
+++ b/x-pack/plugin/security/cli/build.gradle
@@ -0,0 +1,20 @@
+apply plugin: 'elasticsearch.build'
+
+archivesBaseName = 'elasticsearch-security-cli'
+
+dependencies {
+    compileOnly "org.elasticsearch:elasticsearch:${version}"
+    compileOnly xpackProject('plugin:core')
+    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
+    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
+    testImplementation 'com.google.jimfs:jimfs:1.1'
+    testCompile "junit:junit:${versions.junit}"
+    testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+    testCompile 'org.elasticsearch:securemock:1.2'
+    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+dependencyLicenses {
+    mapping from: /bc.*/, to: 'bouncycastle'
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/licenses/bcpkix-jdk15on-1.59.jar.sha1 b/x-pack/plugin/security/cli/licenses/bcpkix-jdk15on-1.59.jar.sha1
similarity index 100%
rename from x-pack/plugin/core/licenses/bcpkix-jdk15on-1.59.jar.sha1
rename to x-pack/plugin/security/cli/licenses/bcpkix-jdk15on-1.59.jar.sha1
diff --git a/x-pack/plugin/core/licenses/bcprov-jdk15on-1.59.jar.sha1 b/x-pack/plugin/security/cli/licenses/bcprov-jdk15on-1.59.jar.sha1
similarity index 100%
rename from x-pack/plugin/core/licenses/bcprov-jdk15on-1.59.jar.sha1
rename to x-pack/plugin/security/cli/licenses/bcprov-jdk15on-1.59.jar.sha1
diff --git a/x-pack/plugin/core/licenses/bouncycastle-LICENSE.txt b/x-pack/plugin/security/cli/licenses/bouncycastle-LICENSE.txt
similarity index 100%
rename from x-pack/plugin/core/licenses/bouncycastle-LICENSE.txt
rename to x-pack/plugin/security/cli/licenses/bouncycastle-LICENSE.txt
diff --git a/x-pack/plugin/core/licenses/bouncycastle-NOTICE.txt b/x-pack/plugin/security/cli/licenses/bouncycastle-NOTICE.txt
similarity index 100%
rename from x-pack/plugin/core/licenses/bouncycastle-NOTICE.txt
rename to x-pack/plugin/security/cli/licenses/bouncycastle-NOTICE.txt
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertGenUtils.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertGenUtils.java
similarity index 93%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertGenUtils.java
rename to x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertGenUtils.java
index 6273456aca2..0b88f3da40a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertGenUtils.java
+++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertGenUtils.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ssl;
+package org.elasticsearch.xpack.security.cli;
 
 import org.bouncycastle.asn1.ASN1Encodable;
 import org.bouncycastle.asn1.ASN1ObjectIdentifier;
@@ -78,7 +78,7 @@ public class CertGenUtils {
      * Generates a CA certificate
      */
     public static X509Certificate generateCACertificate(X500Principal x500Principal, KeyPair keyPair, int days)
-            throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
+        throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
         return generateSignedCertificate(x500Principal, null, keyPair, null, null, true, days, null);
     }
 
@@ -100,7 +100,7 @@ public class CertGenUtils {
      */
     public static X509Certificate generateSignedCertificate(X500Principal principal, GeneralNames subjectAltNames, KeyPair keyPair,
                                                             X509Certificate caCert, PrivateKey caPrivKey, int days)
-            throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
+        throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
         return generateSignedCertificate(principal, subjectAltNames, keyPair, caCert, caPrivKey, false, days, null);
     }
 
@@ -125,7 +125,7 @@ public class CertGenUtils {
     public static X509Certificate generateSignedCertificate(X500Principal principal, GeneralNames subjectAltNames, KeyPair keyPair,
                                                             X509Certificate caCert, PrivateKey caPrivKey,
                                                             int days, String signatureAlgorithm)
-            throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
+        throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
         return generateSignedCertificate(principal, subjectAltNames, keyPair, caCert, caPrivKey, false, days, signatureAlgorithm);
     }
 
@@ -150,7 +150,7 @@ public class CertGenUtils {
     private static X509Certificate generateSignedCertificate(X500Principal principal, GeneralNames subjectAltNames, KeyPair keyPair,
                                                              X509Certificate caCert, PrivateKey caPrivKey, boolean isCa,
                                                              int days, String signatureAlgorithm)
-            throws NoSuchAlgorithmException, CertificateException, CertIOException, OperatorCreationException {
+        throws NoSuchAlgorithmException, CertificateException, CertIOException, OperatorCreationException {
         Objects.requireNonNull(keyPair, "Key-Pair must not be null");
         final DateTime notBefore = new DateTime(DateTimeZone.UTC);
         if (days < 1) {
@@ -175,8 +175,8 @@ public class CertGenUtils {
         }
 
         JcaX509v3CertificateBuilder builder =
-                new JcaX509v3CertificateBuilder(issuer, serial,
-                        new Time(notBefore.toDate(), Locale.ROOT), new Time(notAfter.toDate(), Locale.ROOT), subject, keyPair.getPublic());
+            new JcaX509v3CertificateBuilder(issuer, serial,
+                new Time(notBefore.toDate(), Locale.ROOT), new Time(notAfter.toDate(), Locale.ROOT), subject, keyPair.getPublic());
 
         builder.addExtension(Extension.subjectKeyIdentifier, false, extUtils.createSubjectKeyIdentifier(keyPair.getPublic()));
         builder.addExtension(Extension.authorityKeyIdentifier, false, authorityKeyIdentifier);
@@ -187,8 +187,8 @@ public class CertGenUtils {
 
         PrivateKey signingKey = caPrivKey != null ? caPrivKey : keyPair.getPrivate();
         ContentSigner signer = new JcaContentSignerBuilder(
-                (Strings.isNullOrEmpty(signatureAlgorithm)) ? getDefaultSignatureAlgorithm(signingKey) : signatureAlgorithm)
-                .setProvider(CertGenUtils.BC_PROV).build(signingKey);
+            (Strings.isNullOrEmpty(signatureAlgorithm)) ? getDefaultSignatureAlgorithm(signingKey) : signatureAlgorithm)
+            .setProvider(CertGenUtils.BC_PROV).build(signingKey);
         X509CertificateHolder certificateHolder = builder.build(signer);
         return new JcaX509CertificateConverter().getCertificate(certificateHolder);
     }
@@ -214,7 +214,7 @@ public class CertGenUtils {
                 break;
             default:
                 throw new IllegalArgumentException("Unsupported algorithm : " + key.getAlgorithm()
-                        + " for signature, allowed values for private key algorithm are [RSA, DSA, EC]");
+                    + " for signature, allowed values for private key algorithm are [RSA, DSA, EC]");
         }
         return signatureAlgorithm;
     }
@@ -229,7 +229,7 @@ public class CertGenUtils {
      * @return a certificate signing request
      */
     static PKCS10CertificationRequest generateCSR(KeyPair keyPair, X500Principal principal, GeneralNames sanList)
-            throws IOException, OperatorCreationException {
+        throws IOException, OperatorCreationException {
         Objects.requireNonNull(keyPair, "Key-Pair must not be null");
         Objects.requireNonNull(keyPair.getPublic(), "Public-Key must not be null");
         Objects.requireNonNull(principal, "Principal must not be null");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java
similarity index 90%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateTool.java
rename to x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java
index 5515978c3ca..809e4a6d305 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateTool.java
+++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ssl;
+package org.elasticsearch.xpack.security.cli;
 
 import joptsimple.ArgumentAcceptingOptionSpec;
 import joptsimple.OptionSet;
@@ -34,6 +34,8 @@ import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
 
 import javax.security.auth.x500.X500Principal;
 
@@ -68,6 +70,7 @@ import java.util.zip.ZipOutputStream;
 
 /**
  * CLI tool to make generation of certificates or certificate requests easier for users
+ *
  * @deprecated Replaced by {@link CertificateTool}
  */
 @Deprecated
@@ -81,7 +84,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
     private static final int FILE_EXTENSION_LENGTH = 4;
     static final int MAX_FILENAME_LENGTH = 255 - FILE_EXTENSION_LENGTH;
     private static final Pattern ALLOWED_FILENAME_CHAR_PATTERN =
-            Pattern.compile("[a-zA-Z0-9!@#$%^&{}\\[\\]()_+\\-=,.~'` ]{1," + MAX_FILENAME_LENGTH + "}");
+        Pattern.compile("[a-zA-Z0-9!@#$%^&{}\\[\\]()_+\\-=,.~'` ]{1," + MAX_FILENAME_LENGTH + "}");
     private static final int DEFAULT_KEY_SIZE = 2048;
     private static final BouncyCastleProvider BC_PROV = new BouncyCastleProvider();
 
@@ -96,11 +99,11 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
         // of the parser in this class so that we can defer initialization until after logging has been initialized
         static {
             @SuppressWarnings("unchecked") final ConstructingObjectParser<CertificateInformation, Void> instanceParser =
-                    new ConstructingObjectParser<>(
-                            "instances",
-                            a -> new CertificateInformation(
-                                    (String) a[0], (String) (a[1] == null ? a[0] : a[1]),
-                                    (List<String>) a[2], (List<String>) a[3], (List<String>) a[4]));
+                new ConstructingObjectParser<>(
+                    "instances",
+                    a -> new CertificateInformation(
+                        (String) a[0], (String) (a[1] == null ? a[0] : a[1]),
+                        (List<String>) a[2], (List<String>) a[3], (List<String>) a[4]));
             instanceParser.declareString(ConstructingObjectParser.constructorArg(), new ParseField("name"));
             instanceParser.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("filename"));
             instanceParser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), new ParseField("ip"));
@@ -125,29 +128,29 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
     CertificateGenerateTool() {
         super(DESCRIPTION);
         outputPathSpec = parser.accepts("out", "path of the zip file that the output should be written to")
-                .withRequiredArg();
+            .withRequiredArg();
         csrSpec = parser.accepts("csr", "only generate certificate signing requests");
         caCertPathSpec = parser.accepts("cert", "path to an existing ca certificate").availableUnless(csrSpec).withRequiredArg();
         caKeyPathSpec = parser.accepts("key", "path to an existing ca private key")
-                .availableIf(caCertPathSpec)
-                .requiredIf(caCertPathSpec)
-                .withRequiredArg();
+            .availableIf(caCertPathSpec)
+            .requiredIf(caCertPathSpec)
+            .withRequiredArg();
         caPasswordSpec = parser.accepts("pass", "password for an existing ca private key or the generated ca private key")
-                .availableUnless(csrSpec)
-                .withOptionalArg();
+            .availableUnless(csrSpec)
+            .withOptionalArg();
         caDnSpec = parser.accepts("dn", "distinguished name to use for the generated ca. defaults to " + AUTO_GEN_CA_DN)
-                .availableUnless(caCertPathSpec)
-                .availableUnless(csrSpec)
-                .withRequiredArg();
+            .availableUnless(caCertPathSpec)
+            .availableUnless(csrSpec)
+            .withRequiredArg();
         keysizeSpec = parser.accepts("keysize", "size in bits of RSA keys").withRequiredArg().ofType(Integer.class);
         inputFileSpec = parser.accepts("in", "file containing details of the instances in yaml format").withRequiredArg();
         daysSpec = parser.accepts("days", "number of days that the generated certificates are valid")
-                .availableUnless(csrSpec)
-                .withRequiredArg()
-                .ofType(Integer.class);
+            .availableUnless(csrSpec)
+            .withRequiredArg()
+            .ofType(Integer.class);
         p12Spec = parser.accepts("p12", "output a p12 (PKCS#12) version for each certificate/key pair, with optional password")
-                .availableUnless(csrSpec)
-                .withOptionalArg();
+            .availableUnless(csrSpec)
+            .withOptionalArg();
     }
 
     public static void main(String[] args) throws Exception {
@@ -178,7 +181,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
                 p12Password = null;
             }
             CAInfo caInfo = getCAInfo(terminal, dn, caCertPathSpec.value(options), caKeyPathSpec.value(options), keyPass, prompt, env,
-                    keysize, days);
+                keysize, days);
             Collection<CertificateInformation> certificateInformations = getCertificateInformationList(terminal, inputFile);
             generateAndWriteSignedCertificates(outputFile, certificateInformations, caInfo, keysize, days, p12Password);
         }
@@ -197,7 +200,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
     /**
      * Checks for output file in the user specified options or prompts the user for the output file
      *
-     * @param terminal terminal to communicate with a user
+     * @param terminal   terminal to communicate with a user
      * @param outputPath user specified output file, may be {@code null}
      * @return a {@link Path} to the output file
      */
@@ -223,12 +226,13 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
     /**
      * This method handles the collection of information about each instance that is necessary to generate a certificate. The user may
      * be prompted or the information can be gathered from a file
-     * @param terminal the terminal to use for user interaction
+     *
+     * @param terminal  the terminal to use for user interaction
      * @param inputFile an optional file that will be used to load the instance information
      * @return a {@link Collection} of {@link CertificateInformation} that represents each instance
      */
     static Collection<CertificateInformation> getCertificateInformationList(Terminal terminal, String inputFile)
-            throws Exception {
+        throws Exception {
         if (inputFile != null) {
             return parseAndValidateFile(terminal, resolvePath(inputFile).toAbsolutePath());
         }
@@ -239,7 +243,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
             if (name.isEmpty() == false) {
                 final boolean isNameValidFilename = Name.isValidFilename(name);
                 String filename = terminal.readText("Enter name for directories and files " + (isNameValidFilename ? "[" + name + "]" : "")
-                        + ": " );
+                    + ": ");
                 if (filename.isEmpty() && isNameValidFilename) {
                     filename = name;
                 }
@@ -267,7 +271,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
             }
 
             String exit = terminal.readText("Would you like to specify another instance? Press 'y' to continue entering instance " +
-                    "information: ");
+                "information: ");
             if ("y".equals(exit) == false) {
                 done = true;
             }
@@ -283,7 +287,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
             if (errors.size() > 0) {
                 hasError = true;
                 terminal.println(Terminal.Verbosity.SILENT, "Configuration for instance " + certInfo.name.originalName
-                        + " has invalid details");
+                    + " has invalid details");
                 for (String message : errors) {
                     terminal.println(Terminal.Verbosity.SILENT, " * " + message);
                 }
@@ -298,6 +302,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
 
     /**
      * Parses the input file to retrieve the certificate information
+     *
      * @param file the file to parse
      * @return a collection of certificate information
      */
@@ -305,22 +310,23 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
         try (Reader reader = Files.newBufferedReader(file)) {
             // EMPTY is safe here because we never use namedObject
             XContentParser xContentParser = XContentType.YAML.xContent()
-                    .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, reader);
+                .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, reader);
             return InputFileParser.PARSER.parse(xContentParser, new ArrayList<>(), null);
         }
     }
 
     /**
      * Generates certificate signing requests and writes them out to the specified file in zip format
+     *
      * @param outputFile the file to write the output to. This file must not already exist
-     * @param certInfo the details to use in the certificate signing requests
+     * @param certInfo   the details to use in the certificate signing requests
      */
     static void generateAndWriteCsrs(Path outputFile, Collection<CertificateInformation> certInfo, int keysize) throws Exception {
         fullyWriteFile(outputFile, (outputStream, pemWriter) -> {
             for (CertificateInformation certificateInformation : certInfo) {
                 KeyPair keyPair = CertGenUtils.generateKeyPair(keysize);
                 GeneralNames sanList = getSubjectAlternativeNamesValue(certificateInformation.ipAddresses, certificateInformation.dnsNames,
-                        certificateInformation.commonNames);
+                    certificateInformation.commonNames);
                 PKCS10CertificationRequest csr = CertGenUtils.generateCSR(keyPair, certificateInformation.name.x500Principal, sanList);
 
                 final String dirName = certificateInformation.name.filename + "/";
@@ -347,15 +353,15 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
      * Returns the CA certificate and private key that will be used to sign certificates. These may be specified by the user or
      * automatically generated
      *
-     * @param terminal the terminal to use for prompting the user
-     * @param dn the distinguished name to use for the CA
+     * @param terminal   the terminal to use for prompting the user
+     * @param dn         the distinguished name to use for the CA
      * @param caCertPath the path to the CA certificate or {@code null} if not provided
-     * @param caKeyPath the path to the CA private key or {@code null} if not provided
-     * @param prompt whether we should prompt the user for a password
-     * @param keyPass the password to the private key. If not present and the key is encrypted the user will be prompted
-     * @param env the environment for this tool to resolve files with
-     * @param keysize the size of the key in bits
-     * @param days the number of days that the certificate should be valid for
+     * @param caKeyPath  the path to the CA private key or {@code null} if not provided
+     * @param prompt     whether we should prompt the user for a password
+     * @param keyPass    the password to the private key. If not present and the key is encrypted the user will be prompted
+     * @param env        the environment for this tool to resolve files with
+     * @param keysize    the size of the key in bits
+     * @param days       the number of days that the certificate should be valid for
      * @return CA cert and private key
      */
     static CAInfo getCAInfo(Terminal terminal, String dn, String caCertPath, String caKeyPath, char[] keyPass, boolean prompt,
@@ -366,7 +372,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
             Certificate[] certificates = CertParsingUtils.readCertificates(Collections.singletonList(resolvedCaCertPath), env);
             if (certificates.length != 1) {
                 throw new IllegalArgumentException("expected a single certificate in file [" + caCertPath + "] but found [" +
-                        certificates.length + "]");
+                    certificates.length + "]");
             }
             Certificate caCert = certificates[0];
             PrivateKey privateKey = readPrivateKey(caKeyPath, keyPass, terminal, prompt);
@@ -388,11 +394,12 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
 
     /**
      * Generates signed certificates in PEM format stored in a zip file
-     * @param outputFile the file that the certificates will be written to. This file must not exist
+     *
+     * @param outputFile              the file that the certificates will be written to. This file must not exist
      * @param certificateInformations details for creation of the certificates
-     * @param caInfo the CA information to sign the certificates with
-     * @param keysize the size of the key in bits
-     * @param days the number of days that the certificate should be valid for
+     * @param caInfo                  the CA information to sign the certificates with
+     * @param keysize                 the size of the key in bits
+     * @param days                    the number of days that the certificate should be valid for
      */
     static void generateAndWriteSignedCertificates(Path outputFile, Collection<CertificateInformation> certificateInformations,
                                                    CAInfo caInfo, int keysize, int days, char[] pkcs12Password) throws Exception {
@@ -403,9 +410,9 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
             for (CertificateInformation certificateInformation : certificateInformations) {
                 KeyPair keyPair = CertGenUtils.generateKeyPair(keysize);
                 Certificate certificate = CertGenUtils.generateSignedCertificate(certificateInformation.name.x500Principal,
-                        getSubjectAlternativeNamesValue(certificateInformation.ipAddresses, certificateInformation.dnsNames,
-                                certificateInformation.commonNames),
-                        keyPair, caInfo.caCert, caInfo.privateKey, days);
+                    getSubjectAlternativeNamesValue(certificateInformation.ipAddresses, certificateInformation.dnsNames,
+                        certificateInformation.commonNames),
+                    keyPair, caInfo.caCert, caInfo.privateKey, days);
 
                 final String dirName = certificateInformation.name.filename + "/";
                 ZipEntry zipEntry = new ZipEntry(dirName);
@@ -429,7 +436,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
                     final KeyStore pkcs12 = KeyStore.getInstance("PKCS12");
                     pkcs12.load(null);
                     pkcs12.setKeyEntry(certificateInformation.name.originalName, keyPair.getPrivate(), pkcs12Password,
-                            new Certificate[]{certificate});
+                        new Certificate[]{certificate});
 
                     outputStream.putNextEntry(new ZipEntry(entryBase + ".p12"));
                     pkcs12.store(outputStream, pkcs12Password);
@@ -441,7 +448,8 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
 
     /**
      * This method handles the deletion of a file in the case of a partial write
-     * @param file the file that is being written to
+     *
+     * @param file   the file that is being written to
      * @param writer writes the contents of the file
      */
     private static void fullyWriteFile(Path file, Writer writer) throws Exception {
@@ -468,9 +476,10 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
     /**
      * This method handles writing out the certificate authority cert and private key if the certificate authority was generated by
      * this invocation of the tool
+     *
      * @param outputStream the output stream to write to
-     * @param pemWriter the writer for PEM objects
-     * @param info the certificate authority information
+     * @param pemWriter    the writer for PEM objects
+     * @param info         the certificate authority information
      */
     private static void writeCAInfoIfGenerated(ZipOutputStream outputStream, JcaPEMWriter pemWriter, CAInfo info) throws Exception {
         if (info.generated) {
@@ -577,14 +586,15 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
     /**
      * Helper method to read a private key and support prompting of user for a key. To avoid passwords being placed as an argument we
      * can prompt the user for their password if we encounter an encrypted key.
-     * @param path the path to the private key
+     *
+     * @param path     the path to the private key
      * @param password the password provided by the user or {@code null}
      * @param terminal the terminal to use for user interaction
-     * @param prompt whether to prompt the user or not
+     * @param prompt   whether to prompt the user or not
      * @return the {@link PrivateKey} that was read from the file
      */
     private static PrivateKey readPrivateKey(String path, char[] password, Terminal terminal, boolean prompt)
-                                            throws Exception {
+        throws Exception {
         AtomicReference<char[]> passwordReference = new AtomicReference<>(password);
         try {
             return PemUtils.readPrivateKey(resolvePath(path), () -> {
@@ -682,7 +692,7 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
                 }
             } catch (IllegalArgumentException e) {
                 String error = "[" + name + "] could not be converted to a valid DN\n" + e.getMessage() + "\n"
-                        + ExceptionsHelper.stackTrace(e);
+                    + ExceptionsHelper.stackTrace(e);
                 return new Name(name, null, null, error);
             }
 
@@ -695,15 +705,15 @@ public class CertificateGenerateTool extends EnvironmentAwareCommand {
 
         static boolean isValidFilename(String name) {
             return ALLOWED_FILENAME_CHAR_PATTERN.matcher(name).matches()
-                    && ALLOWED_FILENAME_CHAR_PATTERN.matcher(resolvePath(name).toString()).matches()
-                    && name.startsWith(".") == false;
+                && ALLOWED_FILENAME_CHAR_PATTERN.matcher(resolvePath(name).toString()).matches()
+                && name.startsWith(".") == false;
         }
 
         @Override
         public String toString() {
             return getClass().getSimpleName()
-                    + "{original=[" + originalName + "] principal=[" + x500Principal
-                    + "] file=[" + filename + "] err=[" + error + "]}";
+                + "{original=[" + originalName + "] principal=[" + x500Principal
+                + "] file=[" + filename + "] err=[" + error + "]}";
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertificateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java
similarity index 92%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertificateTool.java
rename to x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java
index dd90df4dd6a..a966cac9109 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertificateTool.java
+++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ssl;
+package org.elasticsearch.xpack.security.cli;
 
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
@@ -39,6 +39,8 @@ import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
 
 import javax.security.auth.x500.X500Principal;
 
@@ -101,7 +103,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
     private static final int FILE_EXTENSION_LENGTH = 4;
     static final int MAX_FILENAME_LENGTH = 255 - FILE_EXTENSION_LENGTH;
     private static final Pattern ALLOWED_FILENAME_CHAR_PATTERN =
-            Pattern.compile("[a-zA-Z0-9!@#$%^&{}\\[\\]()_+\\-=,.~'` ]{1," + MAX_FILENAME_LENGTH + "}");
+        Pattern.compile("[a-zA-Z0-9!@#$%^&{}\\[\\]()_+\\-=,.~'` ]{1," + MAX_FILENAME_LENGTH + "}");
     private static final int DEFAULT_KEY_SIZE = 2048;
 
     /**
@@ -115,11 +117,11 @@ public class CertificateTool extends LoggingAwareMultiCommand {
         // of the parser in this class so that we can defer initialization until after logging has been initialized
         static {
             @SuppressWarnings("unchecked") final ConstructingObjectParser<CertificateInformation, Void> instanceParser =
-                    new ConstructingObjectParser<>(
-                            "instances",
-                            a -> new CertificateInformation(
-                                    (String) a[0], (String) (a[1] == null ? a[0] : a[1]),
-                                    (List<String>) a[2], (List<String>) a[3], (List<String>) a[4]));
+                new ConstructingObjectParser<>(
+                    "instances",
+                    a -> new CertificateInformation(
+                        (String) a[0], (String) (a[1] == null ? a[0] : a[1]),
+                        (List<String>) a[2], (List<String>) a[3], (List<String>) a[4]));
             instanceParser.declareString(ConstructingObjectParser.constructorArg(), new ParseField("name"));
             instanceParser.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("filename"));
             instanceParser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), new ParseField("ip"));
@@ -144,28 +146,28 @@ public class CertificateTool extends LoggingAwareMultiCommand {
 
 
     static final String INTRO_TEXT = "This tool assists you in the generation of X.509 certificates and certificate\n" +
-            "signing requests for use with SSL/TLS in the Elastic stack.";
+        "signing requests for use with SSL/TLS in the Elastic stack.";
 
     static final String INSTANCE_EXPLANATION =
-            "    * An instance is any piece of the Elastic Stack that requires a SSL certificate.\n" +
-                    "      Depending on your configuration, Elasticsearch, Logstash, Kibana, and Beats\n" +
-                    "      may all require a certificate and private key.\n" +
-                    "    * The minimum required value for each instance is a name. This can simply be the\n" +
-                    "      hostname, which will be used as the Common Name of the certificate. A full\n" +
-                    "      distinguished name may also be used.\n" +
-                    "    * A filename value may be required for each instance. This is necessary when the\n" +
-                    "      name would result in an invalid file or directory name. The name provided here\n" +
-                    "      is used as the directory name (within the zip) and the prefix for the key and\n" +
-                    "      certificate files. The filename is required if you are prompted and the name\n" +
-                    "      is not displayed in the prompt.\n" +
-                    "    * IP addresses and DNS names are optional. Multiple values can be specified as a\n" +
-                    "      comma separated string. If no IP addresses or DNS names are provided, you may\n" +
-                    "      disable hostname verification in your SSL configuration.";
+        "    * An instance is any piece of the Elastic Stack that requires a SSL certificate.\n" +
+            "      Depending on your configuration, Elasticsearch, Logstash, Kibana, and Beats\n" +
+            "      may all require a certificate and private key.\n" +
+            "    * The minimum required value for each instance is a name. This can simply be the\n" +
+            "      hostname, which will be used as the Common Name of the certificate. A full\n" +
+            "      distinguished name may also be used.\n" +
+            "    * A filename value may be required for each instance. This is necessary when the\n" +
+            "      name would result in an invalid file or directory name. The name provided here\n" +
+            "      is used as the directory name (within the zip) and the prefix for the key and\n" +
+            "      certificate files. The filename is required if you are prompted and the name\n" +
+            "      is not displayed in the prompt.\n" +
+            "    * IP addresses and DNS names are optional. Multiple values can be specified as a\n" +
+            "      comma separated string. If no IP addresses or DNS names are provided, you may\n" +
+            "      disable hostname verification in your SSL configuration.";
 
     static final String CA_EXPLANATION =
-            "    * All certificates generated by this tool will be signed by a certificate authority (CA).\n" +
-                    "    * The tool can automatically generate a new CA for you, or you can provide your own with the\n" +
-                    "         -ca or -ca-cert command line options.";
+        "    * All certificates generated by this tool will be signed by a certificate authority (CA).\n" +
+            "    * The tool can automatically generate a new CA for you, or you can provide your own with the\n" +
+            "         -ca or -ca-cert command line options.";
 
 
     abstract static class CertificateCommand extends EnvironmentAwareCommand {
@@ -202,32 +204,32 @@ public class CertificateTool extends LoggingAwareMultiCommand {
         final void acceptCertificateGenerationOptions() {
             pemFormatSpec = parser.accepts("pem", "output certificates and keys in PEM format instead of PKCS#12");
             daysSpec = parser.accepts("days", "number of days that the generated certificates are valid")
-                    .withRequiredArg().ofType(Integer.class);
+                .withRequiredArg().ofType(Integer.class);
         }
 
         final void acceptsCertificateAuthority() {
             caPkcs12PathSpec = parser.accepts("ca", "path to an existing ca key pair (in PKCS#12 format)").withRequiredArg();
             caCertPathSpec = parser.accepts("ca-cert", "path to an existing ca certificate")
-                    .availableUnless(caPkcs12PathSpec)
-                    .withRequiredArg();
+                .availableUnless(caPkcs12PathSpec)
+                .withRequiredArg();
             caKeyPathSpec = parser.accepts("ca-key", "path to an existing ca private key")
-                    .availableIf(caCertPathSpec)
-                    .requiredIf(caCertPathSpec)
-                    .withRequiredArg();
+                .availableIf(caCertPathSpec)
+                .requiredIf(caCertPathSpec)
+                .withRequiredArg();
 
             keepCaKeySpec = parser.accepts("keep-ca-key", "retain the CA private key for future use")
-                    .availableUnless(caPkcs12PathSpec)
-                    .availableUnless(caCertPathSpec);
+                .availableUnless(caPkcs12PathSpec)
+                .availableUnless(caCertPathSpec);
 
             caPasswordSpec = parser.accepts("ca-pass", "password for an existing ca private key or the generated ca private key")
-                    .withOptionalArg();
+                .withOptionalArg();
 
             acceptsCertificateAuthorityName();
         }
 
         void acceptsCertificateAuthorityName() {
             OptionSpecBuilder builder = parser.accepts("ca-dn",
-                    "distinguished name to use for the generated ca. defaults to " + AUTO_GEN_CA_DN);
+                "distinguished name to use for the generated ca. defaults to " + AUTO_GEN_CA_DN);
             if (caPkcs12PathSpec != null) {
                 builder = builder.availableUnless(caPkcs12PathSpec);
             }
@@ -336,11 +338,11 @@ public class CertificateTool extends LoggingAwareMultiCommand {
             char[] passwordOption = getChars(caPasswordSpec.value(options));
 
             Map<Certificate, Key> keys = withPassword("CA (" + path + ")", passwordOption,
-                    terminal, password -> CertParsingUtils.readPkcs12KeyPairs(path, password, a -> password));
+                terminal, password -> CertParsingUtils.readPkcs12KeyPairs(path, password, a -> password));
 
             if (keys.size() != 1) {
                 throw new IllegalArgumentException("expected a single key in file [" + path.toAbsolutePath() + "] but found [" +
-                        keys.size() + "]");
+                    keys.size() + "]");
             }
             final Map.Entry<Certificate, Key> pair = keys.entrySet().iterator().next();
             return new CAInfo((X509Certificate) pair.getKey(), (PrivateKey) pair.getValue());
@@ -358,7 +360,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
             Certificate[] certificates = CertParsingUtils.readCertificates(Collections.singletonList(resolvedCaCertPath), env);
             if (certificates.length != 1) {
                 throw new IllegalArgumentException("expected a single certificate in file [" + resolvedCaCertPath + "] but found [" +
-                        certificates.length + "]");
+                    certificates.length + "]");
             }
             X509Certificate caCert = (X509Certificate) certificates[0];
             PrivateKey privateKey = readPrivateKey(key, getChars(password), terminal);
@@ -391,7 +393,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
          * @return a {@link Collection} of {@link CertificateInformation} that represents each instance
          */
         Collection<CertificateInformation> getCertificateInformationList(Terminal terminal, OptionSet options)
-                throws Exception {
+            throws Exception {
             final Path input = resolvePath(options, inputFileSpec);
             if (input != null) {
                 return parseAndValidateFile(terminal, input.toAbsolutePath());
@@ -456,7 +458,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
                 }
 
                 String exit = terminal.readText("Would you like to specify another instance? Press 'y' to continue entering instance " +
-                        "information: ");
+                    "information: ");
                 if ("y".equals(exit) == false) {
                     done = true;
                 }
@@ -468,7 +470,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
             final boolean isNameValidFilename = Name.isValidFilename(certName);
             while (true) {
                 String filename = terminal.readText("Enter name for directories and files of " + certName +
-                        (isNameValidFilename ? " [" + certName + "]" : "") + ": ");
+                    (isNameValidFilename ? " [" + certName + "]" : "") + ": ");
                 if (filename.isEmpty() && isNameValidFilename) {
                     return certName;
                 }
@@ -490,7 +492,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
          * @param includeKey   if true, write the CA key in PEM format
          */
         static void writeCAInfo(ZipOutputStream outputStream, JcaPEMWriter pemWriter, CAInfo info, boolean includeKey)
-                throws Exception {
+            throws Exception {
             final String caDirName = createCaDirectory(outputStream);
             outputStream.putNextEntry(new ZipEntry(caDirName + "ca.crt"));
             pemWriter.writeObject(info.certAndKey.cert);
@@ -546,7 +548,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
             pkcs12.load(null);
             withPassword(fileName, password, terminal, p12Password -> {
                 if (isAscii(p12Password)) {
-                    pkcs12.setKeyEntry(alias, pair.key, p12Password, new Certificate[] { pair.cert });
+                    pkcs12.setKeyEntry(alias, pair.key, p12Password, new Certificate[]{pair.cert});
                     if (caCert != null) {
                         pkcs12.setCertificateEntry("ca", caCert);
                     }
@@ -574,7 +576,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
             terminal.println("The 'csr' mode generates certificate signing requests that can be sent to");
             terminal.println("a trusted certificate authority");
             terminal.println("    * By default, this generates a single CSR for a single instance.");
-            terminal.println("    * You can use the '-multiple' option to generate CSRs for multiple" );
+            terminal.println("    * You can use the '-multiple' option to generate CSRs for multiple");
             terminal.println("       instances, each with their own private key.");
             terminal.println("    * The '-in' option allows for the CSR generation to be automated");
             terminal.println("       by describing the details of each instance in a YAML file");
@@ -616,7 +618,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
                 for (CertificateInformation certificateInformation : certInfo) {
                     KeyPair keyPair = CertGenUtils.generateKeyPair(keySize);
                     GeneralNames sanList = getSubjectAlternativeNamesValue(certificateInformation.ipAddresses,
-                            certificateInformation.dnsNames, certificateInformation.commonNames);
+                        certificateInformation.dnsNames, certificateInformation.commonNames);
                     PKCS10CertificationRequest csr = CertGenUtils.generateCSR(keyPair, certificateInformation.name.x500Principal, sanList);
 
                     final String dirName = certificateInformation.name.filename + "/";
@@ -750,7 +752,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
          */
         void generateAndWriteSignedCertificates(Path output, boolean writeZipFile, OptionSet options,
                                                 Collection<CertificateInformation> certs, CAInfo caInfo, Terminal terminal)
-                throws Exception {
+            throws Exception {
 
             checkDirectory(output, terminal);
 
@@ -805,7 +807,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
                             final String fileName = entryBase + ".p12";
                             outputStream.putNextEntry(new ZipEntry(fileName));
                             writePkcs12(fileName, outputStream, certificateInformation.name.originalName, pair, caInfo.certAndKey.cert,
-                                    outputPassword, terminal);
+                                outputPassword, terminal);
                             outputStream.closeEntry();
                         }
                     }
@@ -815,7 +817,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
                 CertificateInformation certificateInformation = certs.iterator().next();
                 CertificateAndKey pair = generateCertificateAndKey(certificateInformation, caInfo, keySize, days);
                 fullyWriteFile(output, stream -> writePkcs12(output.getFileName().toString(), stream,
-                        certificateInformation.name.originalName, pair, caInfo.certAndKey.cert, outputPassword, terminal));
+                    certificateInformation.name.originalName, pair, caInfo.certAndKey.cert, outputPassword, terminal));
             }
         }
 
@@ -823,9 +825,9 @@ public class CertificateTool extends LoggingAwareMultiCommand {
                                                             int keySize, int days) throws Exception {
             KeyPair keyPair = CertGenUtils.generateKeyPair(keySize);
             Certificate certificate = CertGenUtils.generateSignedCertificate(certificateInformation.name.x500Principal,
-                    getSubjectAlternativeNamesValue(certificateInformation.ipAddresses, certificateInformation.dnsNames,
-                            certificateInformation.commonNames),
-                    keyPair, caInfo.certAndKey.cert, caInfo.certAndKey.key, days);
+                getSubjectAlternativeNamesValue(certificateInformation.ipAddresses, certificateInformation.dnsNames,
+                    certificateInformation.commonNames),
+                keyPair, caInfo.certAndKey.cert, caInfo.certAndKey.key, days);
             return new CertificateAndKey((X509Certificate) certificate, keyPair.getPrivate());
         }
 
@@ -872,7 +874,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
             } else {
                 final String fileName = output.getFileName().toString();
                 fullyWriteFile(output, outputStream ->
-                        writePkcs12(fileName, outputStream, "ca", caInfo.certAndKey, null, caInfo.password, terminal));
+                    writePkcs12(fileName, outputStream, "ca", caInfo.certAndKey, null, caInfo.password, terminal));
             }
         }
     }
@@ -912,7 +914,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
         try (Reader reader = Files.newBufferedReader(file)) {
             // EMPTY is safe here because we never use namedObject
             XContentParser xContentParser = XContentType.YAML.xContent()
-                    .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, reader);
+                .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, reader);
             return CertificateToolParser.PARSER.parse(xContentParser, new ArrayList<>(), null);
         }
     }
@@ -1015,7 +1017,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
      * @return the {@link PrivateKey} that was read from the file
      */
     private static PrivateKey readPrivateKey(Path path, char[] password, Terminal terminal)
-            throws Exception {
+        throws Exception {
         AtomicReference<char[]> passwordReference = new AtomicReference<>(password);
         try {
             return PemUtils.readPrivateKey(path, () -> {
@@ -1125,7 +1127,7 @@ public class CertificateTool extends LoggingAwareMultiCommand {
                 }
             } catch (IllegalArgumentException e) {
                 String error = "[" + name + "] could not be converted to a valid DN\n" + e.getMessage() + "\n"
-                        + ExceptionsHelper.stackTrace(e);
+                    + ExceptionsHelper.stackTrace(e);
                 return new Name(name, null, null, error);
             }
 
@@ -1138,15 +1140,15 @@ public class CertificateTool extends LoggingAwareMultiCommand {
 
         static boolean isValidFilename(String name) {
             return ALLOWED_FILENAME_CHAR_PATTERN.matcher(name).matches()
-                    && ALLOWED_FILENAME_CHAR_PATTERN.matcher(resolvePath(name).toString()).matches()
-                    && name.startsWith(".") == false;
+                && ALLOWED_FILENAME_CHAR_PATTERN.matcher(resolvePath(name).toString()).matches()
+                && name.startsWith(".") == false;
         }
 
         @Override
         public String toString() {
             return getClass().getSimpleName()
-                    + "{original=[" + originalName + "] principal=[" + x500Principal
-                    + "] file=[" + filename + "] err=[" + error + "]}";
+                + "{original=[" + originalName + "] principal=[" + x500Principal
+                + "] file=[" + filename + "] err=[" + error + "]}";
         }
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertGenUtilsTests.java b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertGenUtilsTests.java
similarity index 97%
rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertGenUtilsTests.java
rename to x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertGenUtilsTests.java
index 20259144b42..bb1ed014b9c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertGenUtilsTests.java
+++ b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertGenUtilsTests.java
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.core.ssl;
+package org.elasticsearch.xpack.security.cli;
 
 import org.bouncycastle.asn1.x509.GeneralName;
 import org.bouncycastle.asn1.x509.GeneralNames;
@@ -12,6 +12,7 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.security.cli.CertGenUtils;
 import org.junit.BeforeClass;
 
 import java.math.BigInteger;
diff --git a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateToolTests.java b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateGenerateToolTests.java
similarity index 98%
rename from x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateToolTests.java
rename to x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateGenerateToolTests.java
index dde0b7645df..91fd55933c5 100644
--- a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateGenerateToolTests.java
+++ b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateGenerateToolTests.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ssl;
+package org.elasticsearch.xpack.security.cli;
 
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
@@ -33,9 +33,11 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
-import org.elasticsearch.xpack.core.ssl.CertificateGenerateTool.CAInfo;
-import org.elasticsearch.xpack.core.ssl.CertificateGenerateTool.CertificateInformation;
-import org.elasticsearch.xpack.core.ssl.CertificateGenerateTool.Name;
+import org.elasticsearch.xpack.security.cli.CertificateGenerateTool.CAInfo;
+import org.elasticsearch.xpack.security.cli.CertificateGenerateTool.CertificateInformation;
+import org.elasticsearch.xpack.security.cli.CertificateGenerateTool.Name;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
 import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.BeforeClass;
@@ -359,8 +361,8 @@ public class CertificateGenerateToolTests extends ESTestCase {
 
     public void testGetCAInfo() throws Exception {
         Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
-        Path testNodeCertPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
-        Path testNodeKeyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
+        Path testNodeCertPath = getDataPath("/org/elasticsearch/xpack/security/cli/testnode.crt");
+        Path testNodeKeyPath = getDataPath("/org/elasticsearch/xpack/security/cli/testnode.pem");
         final boolean passwordPrompt = randomBoolean();
         MockTerminal terminal = new MockTerminal();
         if (passwordPrompt) {
diff --git a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateToolTests.java b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateToolTests.java
similarity index 98%
rename from x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateToolTests.java
rename to x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateToolTests.java
index 706d5dbab5f..9e970ea559a 100644
--- a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/core/ssl/CertificateToolTests.java
+++ b/x-pack/plugin/security/cli/src/test/java/org/elasticsearch/xpack/security/cli/CertificateToolTests.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ssl;
+package org.elasticsearch.xpack.security.cli;
 
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
@@ -39,12 +39,14 @@ import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.TestMatchers;
-import org.elasticsearch.xpack.core.ssl.CertificateTool.CAInfo;
-import org.elasticsearch.xpack.core.ssl.CertificateTool.CertificateAuthorityCommand;
-import org.elasticsearch.xpack.core.ssl.CertificateTool.CertificateCommand;
-import org.elasticsearch.xpack.core.ssl.CertificateTool.CertificateInformation;
-import org.elasticsearch.xpack.core.ssl.CertificateTool.GenerateCertificateCommand;
-import org.elasticsearch.xpack.core.ssl.CertificateTool.Name;
+import org.elasticsearch.xpack.security.cli.CertificateTool.CAInfo;
+import org.elasticsearch.xpack.security.cli.CertificateTool.CertificateAuthorityCommand;
+import org.elasticsearch.xpack.security.cli.CertificateTool.CertificateCommand;
+import org.elasticsearch.xpack.security.cli.CertificateTool.CertificateInformation;
+import org.elasticsearch.xpack.security.cli.CertificateTool.GenerateCertificateCommand;
+import org.elasticsearch.xpack.security.cli.CertificateTool.Name;
+import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
+import org.elasticsearch.xpack.core.ssl.PemUtils;
 import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.BeforeClass;
@@ -387,8 +389,8 @@ public class CertificateToolTests extends ESTestCase {
 
     public void testGetCAInfo() throws Exception {
         Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
-        Path testNodeCertPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
-        Path testNodeKeyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem");
+        Path testNodeCertPath = getDataPath("/org/elasticsearch/xpack/security/cli/testnode.crt");
+        Path testNodeKeyPath = getDataPath("/org/elasticsearch/xpack/security/cli/testnode.pem");
         final boolean passwordPrompt = randomBoolean();
         MockTerminal terminal = new MockTerminal();
         if (passwordPrompt) {
diff --git a/x-pack/plugin/security/cli/src/test/resources/org/elasticsearch/xpack/security/cli/testnode.crt b/x-pack/plugin/security/cli/src/test/resources/org/elasticsearch/xpack/security/cli/testnode.crt
new file mode 100644
index 00000000000..08c160bcea5
--- /dev/null
+++ b/x-pack/plugin/security/cli/src/test/resources/org/elasticsearch/xpack/security/cli/testnode.crt
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID0zCCArugAwIBAgIJALi5bDfjMszLMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV
+BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEgMB4GA1UEAxMXRWxhc3Rp
+Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTUwOTIzMTg1MjU3WhcNMTkwOTIyMTg1MjU3
+WjBIMQwwCgYDVQQKEwNvcmcxFjAUBgNVBAsTDWVsYXN0aWNzZWFyY2gxIDAeBgNV
+BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEA3rGZ1QbsW0+MuyrSLmMfDFKtLBkIFW8V0gRuurFg1PUKKNR1
+Mq2tMVwjjYETAU/UY0iKZOzjgvYPKhDTYBTte/WHR1ZK4CYVv7TQX/gtFQG/ge/c
+7u0sLch9p7fbd+/HZiLS/rBEZDIohvgUvzvnA8+OIYnw4kuxKo/5iboAIS41klMg
+/lATm8V71LMY68inht71/ZkQoAHKgcR9z4yNYvQ1WqKG8DG8KROXltll3sTrKbl5
+zJhn660es/1ZnR6nvwt6xnSTl/mNHMjkfv1bs4rJ/py3qPxicdoSIn/KyojUcgHV
+F38fuAy2CQTdjVG5fWj9iz+mQvLm3+qsIYQdFwIDAQABo4G/MIG8MAkGA1UdEwQC
+MAAwHQYDVR0OBBYEFEMMWLWQi/g83PzlHYqAVnty5L7HMIGPBgNVHREEgYcwgYSC
+CWxvY2FsaG9zdIIVbG9jYWxob3N0LmxvY2FsZG9tYWluggpsb2NhbGhvc3Q0ghds
+b2NhbGhvc3Q0LmxvY2FsZG9tYWluNIIKbG9jYWxob3N0NoIXbG9jYWxob3N0Ni5s
+b2NhbGRvbWFpbjaHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQEL
+BQADggEBAMjGGXT8Nt1tbl2GkiKtmiuGE2Ej66YuZ37WSJViaRNDVHLlg87TCcHe
+k2rdO+6sFqQbbzEfwQ05T7xGmVu7tm54HwKMRugoQ3wct0bQC5wEWYN+oMDvSyO6
+M28mZwWb4VtR2IRyWP+ve5DHwTM9mxWa6rBlGzsQqH6YkJpZojzqk/mQTug+Y8aE
+mVoqRIPMHq9ob+S9qd5lp09+MtYpwPfTPx/NN+xMEooXWW/ARfpGhWPkg/FuCu4z
+1tFmCqHgNcWirzMm3dQpF78muE9ng6OB2MXQwL4VgnVkxmlZNHbkR2v/t8MyZJxC
+y4g6cTMM3S/UMt5/+aIB2JAuMKyuD+A=
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/security/cli/src/test/resources/org/elasticsearch/xpack/security/cli/testnode.pem b/x-pack/plugin/security/cli/src/test/resources/org/elasticsearch/xpack/security/cli/testnode.pem
new file mode 100644
index 00000000000..5a67e103344
--- /dev/null
+++ b/x-pack/plugin/security/cli/src/test/resources/org/elasticsearch/xpack/security/cli/testnode.pem
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,9D867F7E0C94D013
+
+dVoVCjPeg1wgS7rVtOvGfQcrZyLkx393aWRnFq45tbjKBVuITtJ9vI7o4QXOV/15
+Gnb6WhXGIdWrzsxEAd46K6hIuNSISd4Emsx6c2Q5hTqWXXfexbOZBNfTtXtdJPnJ
+1jAaikhtztLo3JSLTKNY5sNxd+XbaQyYVUWvueK6zOaIIMETvB+VPVFd9i1ROibk
+Sgdtyj01KjkoalifqK/tA0CIYNKL0S6/eoK3UhAlpIprlpV+cnXa940C6bjLeJPt
+PMAGGp5RrplxSgrSerw3I9DOWkHGtpqzIka3XneNUXJP8k4HUJ+aZkGH2ZILKS8d
+4KMIb+KZSpHEGn+6uGccWLtZZmAjWJrDw56JbQtSHdRYLBRSOjLbTvQoPu/2Hpli
+7HOxbotlvjptMunncq5aqK57SHA1dh0cwF7J3LUmGFJ67eoz+VV3b5qMn4MopSeI
+mS16Ydd3nGpjSrln/elM0CQxqWfcOAXRZpDpFUQoXcBrLVzvz2DBl/0CrTRLhgzi
+CO+5/IVcBWRlYpRNGgjjP7q0j6URID3jk5J06fYQXmBiwQT5j+GZqqzpMCJ9mIy2
+1O9SN1hebJnIcEU+E0njn/MGjlYdPywhaCy8pqElp6Q8TUEJpwLRFO/owCoBet/n
+ZmCXUjfCGhc1pWHufFcDEQ6xMgEWWY/tdwCZeSU7EhErTjCbfupg+55A5fpDml0m
+3wH4CFcuRjlqyx6Ywixm1ATeitDtJl5HQTw6b8OtEXwSgRmZ0eSqSRVk9QbVS7gu
+IpQe09/Zimb5HzjZqZ3fdqHlcW4xax8hyJeyIvF5ZJ57eY8CBvu/wP2GDn26QnvF
+xQqdfDbq1H4JmpwUHpbFwBoQK4Q6WFd1z4EA9bRQeo3H9PoqoOwMDjzajwLRF7b7
+q6tYH/n9PyHwdf1c4fFwgSmL1toXGfKlA9hjIaLsRSDD6srT5EdUk78bsnddwI51
+tu7C7P4JG+h1VdRNMNTlqtileWsIE7Nn2A1OkcUxZdF5mamENpDpJcHePLto6c8q
+FKiwyFMsxhgsj6HK2HqO+UA4sX5Ni4oHwiPmb//EZLn045M5i1AN26KosJmb8++D
+sgR5reWRy+UqJCTYblVg+7Dx++ggUnfxVyQEsWmw5r5f4KU5wXBkvoVMGtPNa9DE
+n/uLtObD1qkNL38pRsr2OGRchYCgEoKGqEISBP4knfGXLOlWiW/246j9QzI97r1u
+tvy7fKg28G7AUz9l6bpewsPHefBUeRQeieP9eJINaEpxkF/w2RpKDLpQjWxwDDOM
+s+D0mrBMJve17AmJ8rMw6dIQPZYNZ88/jz1uQuUwQ2YlbmtZbCG81k9YMFGEU9XS
+cyhJxj8hvYnt2PR5Z9/cJPyWOs0m/ufOeeQQ8SnU/lzmrQnpzUd2Z6p5i/B7LdRP
+n1kX+l1qynuPnjvBz4nJQE0p6nzW8RyCDSniC9mtYtZmhgC8icqxgbvS7uEOBIYJ
+NbK+0bEETTO34iY/JVTIqLOw3iQZYMeUpxpj6Phgx/oooxMTquMecPKNgeVtaBst
+qjTNPX0ti1/HYpZqzYi8SV8YjHSJWCVMsZjKPr3W/HIcCKqYoIfgzi83Ha2KMQx6
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/security/src/main/bin/elasticsearch-certgen b/x-pack/plugin/security/src/main/bin/elasticsearch-certgen
index 4a192ac3b16..8e88e845e02 100644
--- a/x-pack/plugin/security/src/main/bin/elasticsearch-certgen
+++ b/x-pack/plugin/security/src/main/bin/elasticsearch-certgen
@@ -4,7 +4,8 @@
 # or more contributor license agreements. Licensed under the Elastic License;
 # you may not use this file except in compliance with the Elastic License.
 
-ES_MAIN_CLASS=org.elasticsearch.xpack.core.ssl.CertificateGenerateTool \
+ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.CertificateGenerateTool \
   ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
+  ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
   "`dirname "$0"`"/elasticsearch-cli \
   "$@"
diff --git a/x-pack/plugin/security/src/main/bin/elasticsearch-certgen.bat b/x-pack/plugin/security/src/main/bin/elasticsearch-certgen.bat
index b5842b57b16..bb303f740e5 100644
--- a/x-pack/plugin/security/src/main/bin/elasticsearch-certgen.bat
+++ b/x-pack/plugin/security/src/main/bin/elasticsearch-certgen.bat
@@ -7,8 +7,9 @@ rem you may not use this file except in compliance with the Elastic License.
 setlocal enabledelayedexpansion
 setlocal enableextensions
 
-set ES_MAIN_CLASS=org.elasticsearch.xpack.core.ssl.CertificateGenerateTool
+set ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.CertificateGenerateTool
 set ES_ADDITIONAL_SOURCES=x-pack-env;x-pack-security-env
+set ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli
 call "%~dp0elasticsearch-cli.bat" ^
   %%* ^
   || exit /b 1
diff --git a/x-pack/plugin/security/src/main/bin/elasticsearch-certutil b/x-pack/plugin/security/src/main/bin/elasticsearch-certutil
index a13be812f0b..6d94344949b 100644
--- a/x-pack/plugin/security/src/main/bin/elasticsearch-certutil
+++ b/x-pack/plugin/security/src/main/bin/elasticsearch-certutil
@@ -4,7 +4,8 @@
 # or more contributor license agreements. Licensed under the Elastic License;
 # you may not use this file except in compliance with the Elastic License.
 
-ES_MAIN_CLASS=org.elasticsearch.xpack.core.ssl.CertificateTool \
+ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.CertificateTool \
   ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
+  ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
   "`dirname "$0"`"/elasticsearch-cli \
   "$@"
diff --git a/x-pack/plugin/security/src/main/bin/elasticsearch-certutil.bat b/x-pack/plugin/security/src/main/bin/elasticsearch-certutil.bat
index 2e397190f23..34f595824f8 100644
--- a/x-pack/plugin/security/src/main/bin/elasticsearch-certutil.bat
+++ b/x-pack/plugin/security/src/main/bin/elasticsearch-certutil.bat
@@ -7,8 +7,9 @@ rem you may not use this file except in compliance with the Elastic License.
 setlocal enabledelayedexpansion
 setlocal enableextensions
 
-set ES_MAIN_CLASS=org.elasticsearch.xpack.core.ssl.CertificateTool
+set ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.CertificateTool
 set ES_ADDITIONAL_SOURCES=x-pack-env;x-pack-security-env
+set ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli
 call "%~dp0elasticsearch-cli.bat" ^
   %%* ^
   || exit /b 1

From 5525cb1615b7d0a5b4e512305d6e5d2de73f5fcd Mon Sep 17 00:00:00 2001
From: debadair <debadair@elastic.co>
Date: Fri, 20 Jul 2018 14:17:48 -0700
Subject: [PATCH 127/260] [DOCS] Clarified that you must remove X-Pack plugin
 when upgrading from pre-6.3. (#32016)

---
 docs/reference/upgrade/cluster_restart.asciidoc | 6 ++++--
 docs/reference/upgrade/remove-xpack.asciidoc    | 4 ++++
 docs/reference/upgrade/rolling_upgrade.asciidoc | 4 +++-
 3 files changed, 11 insertions(+), 3 deletions(-)
 create mode 100644 docs/reference/upgrade/remove-xpack.asciidoc

diff --git a/docs/reference/upgrade/cluster_restart.asciidoc b/docs/reference/upgrade/cluster_restart.asciidoc
index b092e45ae29..06d5e96f8ef 100644
--- a/docs/reference/upgrade/cluster_restart.asciidoc
+++ b/docs/reference/upgrade/cluster_restart.asciidoc
@@ -47,6 +47,8 @@ include::set-paths-tip.asciidoc[]
 Use the `elasticsearch-plugin` script to install the upgraded version of each
 installed Elasticsearch plugin. All plugins must be upgraded when you upgrade
 a node.
++
+include::remove-xpack.asciidoc[]
 
 . *Start each upgraded node.*
 +
@@ -91,7 +93,7 @@ already have local shard copies.
 +
 --
 When all nodes have joined the cluster and recovered their primary shards,
-reenable allocation by restoring `cluster.routing.allocation.enable` to its 
+reenable allocation by restoring `cluster.routing.allocation.enable` to its
 default:
 
 [source,js]
@@ -123,4 +125,4 @@ GET _cat/recovery
 // CONSOLE
 --
 
-. *Restart machine learning jobs.* 
+. *Restart machine learning jobs.*
diff --git a/docs/reference/upgrade/remove-xpack.asciidoc b/docs/reference/upgrade/remove-xpack.asciidoc
new file mode 100644
index 00000000000..9d4c4c9f779
--- /dev/null
+++ b/docs/reference/upgrade/remove-xpack.asciidoc
@@ -0,0 +1,4 @@
+IMPORTANT: If you use {xpack} and are upgrading from a version prior to 6.3,
+remove {xpack} before restarting: `bin/elasticsearch-plugin remove x-pack`. As
+of 6.3, {xpack} is included in the default distribution. The node will fail to
+start if the old {xpack} plugin is present.
diff --git a/docs/reference/upgrade/rolling_upgrade.asciidoc b/docs/reference/upgrade/rolling_upgrade.asciidoc
index 76a10f752be..e2edb6b2922 100644
--- a/docs/reference/upgrade/rolling_upgrade.asciidoc
+++ b/docs/reference/upgrade/rolling_upgrade.asciidoc
@@ -53,6 +53,8 @@ include::set-paths-tip.asciidoc[]
 Use the `elasticsearch-plugin` script to install the upgraded version of each
 installed Elasticsearch plugin. All plugins must be upgraded when you upgrade
 a node.
++
+include::remove-xpack.asciidoc[]
 
 . *Start the upgraded node.*
 +
@@ -144,7 +146,7 @@ for each node that needs to be updated.
 
 --
 
-. *Restart machine learning jobs.* 
+. *Restart machine learning jobs.*
 
 [IMPORTANT]
 ====================================================

From 042424b43ba0e1bf0748e77f9941ccd3ef578f34 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 20 Jul 2018 17:33:15 -0400
Subject: [PATCH 128/260] Switch full-cluster-restart to new style Requests
 (#32140)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `qa/full-cluster-restart` project to use the new
versions.

It also fixes a small bug in the test for explain on the `_all` field
that was causing it to not properly invoke `_explain`.
---
 .../upgrades/FullClusterRestartIT.java        | 497 +++++++++---------
 1 file changed, 254 insertions(+), 243 deletions(-)

diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index 081a1918674..0b936e44e5b 100644
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.upgrades;
 
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
@@ -34,7 +31,6 @@ import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.test.NotEqualMessageBuilder;
@@ -45,7 +41,6 @@ import org.junit.Before;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Base64;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -142,8 +137,9 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
-            client().performRequest("PUT", "/" + index, Collections.emptyMap(),
-                new StringEntity(Strings.toString(mappingsAndSettings), ContentType.APPLICATION_JSON));
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
 
             count = randomIntBetween(2000, 3000);
             byte[] randomByteArray = new byte[16];
@@ -164,16 +160,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
             count = countOfIndexedRandomDocuments();
         }
 
-        Map<String, String> params = new HashMap<>();
-        params.put("timeout", "2m");
-        params.put("wait_for_status", "green");
-        params.put("wait_for_no_relocating_shards", "true");
-        params.put("wait_for_events", "languid");
-        Map<String, Object> healthRsp = toMap(client().performRequest("GET", "/_cluster/health/" + index, params));
-        logger.info("health api response: {}", healthRsp);
-        assertEquals("green", healthRsp.get("status"));
-        assertFalse((Boolean) healthRsp.get("timed_out"));
-
+        ensureGreenLongWait(index);
         assertBasicSearchWorks(count);
         assertAllSearchWorks(count);
         assertBasicAggregationWorks();
@@ -205,8 +192,9 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
-            client().performRequest("PUT", "/" + index, Collections.emptyMap(),
-                new StringEntity(Strings.toString(mappingsAndSettings), ContentType.APPLICATION_JSON));
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
 
             int numDocs = randomIntBetween(2000, 3000);
             indexRandomDocuments(numDocs, true, false, i -> {
@@ -215,33 +203,26 @@ public class FullClusterRestartIT extends ESRestTestCase {
                     .endObject();
             });
             logger.info("Refreshing [{}]", index);
-            client().performRequest("POST", "/" + index + "/_refresh");
+            client().performRequest(new Request("POST", "/" + index + "/_refresh"));
         } else {
             final int numReplicas = 1;
             final long startTime = System.currentTimeMillis();
             logger.debug("--> creating [{}] replicas for index [{}]", numReplicas, index);
-            String requestBody = "{ \"index\": { \"number_of_replicas\" : " + numReplicas + " }}";
-            Response response = client().performRequest("PUT", "/" + index + "/_settings", Collections.emptyMap(),
-                new StringEntity(requestBody, ContentType.APPLICATION_JSON));
-            assertEquals(200, response.getStatusLine().getStatusCode());
+            Request setNumberOfReplicas = new Request("PUT", "/" + index + "/_settings");
+            setNumberOfReplicas.setJsonEntity("{ \"index\": { \"number_of_replicas\" : " + numReplicas + " }}");
+            Response response = client().performRequest(setNumberOfReplicas);
 
-            Map<String, String> params = new HashMap<>();
-            params.put("timeout", "2m");
-            params.put("wait_for_status", "green");
-            params.put("wait_for_no_relocating_shards", "true");
-            params.put("wait_for_events", "languid");
-            Map<String, Object> healthRsp = toMap(client().performRequest("GET", "/_cluster/health/" + index, params));
-            assertEquals("green", healthRsp.get("status"));
-            assertFalse((Boolean) healthRsp.get("timed_out"));
+            ensureGreenLongWait(index);
 
             logger.debug("--> index [{}] is green, took [{}] ms", index, (System.currentTimeMillis() - startTime));
-            Map<String, Object> recoverRsp = toMap(client().performRequest("GET", "/" + index + "/_recovery"));
+            Map<String, Object> recoverRsp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_recovery")));
             logger.debug("--> recovery status:\n{}", recoverRsp);
 
             Set<Integer> counts = new HashSet<>();
             for (String node : dataNodes(index, client())) {
-                Map<String, Object> responseBody = toMap(client().performRequest("GET", "/" + index + "/_search",
-                    Collections.singletonMap("preference", "_only_nodes:" + node)));
+                Request search = new Request("GET", "/" + index + "/_search");
+                search.addParameter("preference", "_only_nodes:" + node);
+                Map<String, Object> responseBody = entityAsMap(client().performRequest(search));
                 assertNoFailures(responseBody);
                 int hits = (int) XContentMapValues.extractValue("hits.total", responseBody);
                 counts.add(hits);
@@ -282,12 +263,13 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
-            client().performRequest("PUT", "/" + index, Collections.emptyMap(),
-                new StringEntity(Strings.toString(mappingsAndSettings), ContentType.APPLICATION_JSON));
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
 
             String aliasName = "%23" + index; // %23 == #
-            client().performRequest("PUT", "/" + index + "/_alias/" + aliasName);
-            Response response = client().performRequest("HEAD", "/" + index + "/_alias/" + aliasName);
+            client().performRequest(new Request("PUT", "/" + index + "/_alias/" + aliasName));
+            Response response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + aliasName));
             assertEquals(200, response.getStatusLine().getStatusCode());
 
             count = randomIntBetween(32, 128);
@@ -301,19 +283,20 @@ public class FullClusterRestartIT extends ESRestTestCase {
             count = countOfIndexedRandomDocuments();
         }
 
-        logger.error("clusterState=" + toMap(client().performRequest("GET", "/_cluster/state",
-            Collections.singletonMap("metric", "metadata"))));
+        Request request = new Request("GET", "/_cluster/state");
+        request.addParameter("metric", "metadata");
+        logger.error("clusterState=" + entityAsMap(client().performRequest(request)));
         // We can read from the alias just like we can read from the index.
         String aliasName = "%23" + index; // %23 == #
-        Map<String, Object> searchRsp = toMap(client().performRequest("GET", "/" + aliasName + "/_search"));
+        Map<String, Object> searchRsp = entityAsMap(client().performRequest(new Request("GET", "/" + aliasName + "/_search")));
         int totalHits = (int) XContentMapValues.extractValue("hits.total", searchRsp);
         assertEquals(count, totalHits);
         if (runningAgainstOldCluster == false) {
             // We can remove the alias.
-            Response response = client().performRequest("DELETE", "/" + index + "/_alias/" + aliasName);
+            Response response = client().performRequest(new Request("DELETE", "/" + index + "/_alias/" + aliasName));
             assertEquals(200, response.getStatusLine().getStatusCode());
             // and check that it is gone:
-            response = client().performRequest("HEAD", "/" + index + "/_alias/" + aliasName);
+            response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + aliasName));
             assertEquals(404, response.getStatusLine().getStatusCode());
         }
     }
@@ -330,13 +313,14 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
-            client().performRequest("PUT", "/_template/template_1", Collections.emptyMap(),
-                new StringEntity(Strings.toString(mappingsAndSettings), ContentType.APPLICATION_JSON));
-            client().performRequest("PUT", "/" + index);
+            Request createTemplate = new Request("PUT", "/_template/template_1");
+            createTemplate.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createTemplate);
+            client().performRequest(new Request("PUT", "/" + index));
         }
 
         // verifying if we can still read some properties from cluster state api:
-        Map<String, Object> clusterState = toMap(client().performRequest("GET", "/_cluster/state"));
+        Map<String, Object> clusterState = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state")));
 
         // Check some global properties:
         String clusterName = (String) clusterState.get("cluster_name");
@@ -381,8 +365,9 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
-            client().performRequest("PUT", "/" + index, Collections.emptyMap(),
-                new StringEntity(Strings.toString(mappingsAndSettings), ContentType.APPLICATION_JSON));
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
 
             numDocs = randomIntBetween(512, 1024);
             indexRandomDocuments(numDocs, true, true, i -> {
@@ -393,23 +378,20 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
             ensureGreen(index); // wait for source index to be available on both nodes before starting shrink
 
-            String updateSettingsRequestBody = "{\"settings\": {\"index.blocks.write\": true}}";
-            Response rsp = client().performRequest("PUT", "/" + index + "/_settings", Collections.emptyMap(),
-                new StringEntity(updateSettingsRequestBody, ContentType.APPLICATION_JSON));
-            assertEquals(200, rsp.getStatusLine().getStatusCode());
+            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
+            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
+            client().performRequest(updateSettingsRequest);
 
-            String shrinkIndexRequestBody = "{\"settings\": {\"index.number_of_shards\": 1}}";
-            rsp = client().performRequest("PUT", "/" + index + "/_shrink/" + shrunkenIndex, Collections.emptyMap(),
-                new StringEntity(shrinkIndexRequestBody, ContentType.APPLICATION_JSON));
-            assertEquals(200, rsp.getStatusLine().getStatusCode());
+            Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex);
+            shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}");
+            client().performRequest(shrinkIndexRequest);
 
-            rsp = client().performRequest("POST", "/_refresh");
-            assertEquals(200, rsp.getStatusLine().getStatusCode());
+            client().performRequest(new Request("POST", "/_refresh"));
         } else {
             numDocs = countOfIndexedRandomDocuments();
         }
 
-        Map<?, ?> response = toMap(client().performRequest("GET", "/" + index + "/_search"));
+        Map<?, ?> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
         assertNoFailures(response);
         int totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
         assertThat(totalShards, greaterThan(1));
@@ -418,7 +400,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
         int totalHits = (int) XContentMapValues.extractValue("hits.total", response);
         assertEquals(numDocs, totalHits);
 
-        response = toMap(client().performRequest("GET", "/" + shrunkenIndex+ "/_search"));
+        response = entityAsMap(client().performRequest(new Request("GET", "/" + shrunkenIndex+ "/_search")));
         assertNoFailures(response);
         totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
         assertEquals(1, totalShards);
@@ -448,8 +430,9 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
-            client().performRequest("PUT", "/" + index, Collections.emptyMap(),
-                new StringEntity(Strings.toString(mappingsAndSettings), ContentType.APPLICATION_JSON));
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
 
             numDocs = randomIntBetween(512, 1024);
             indexRandomDocuments(numDocs, true, true, i -> {
@@ -460,23 +443,20 @@ public class FullClusterRestartIT extends ESRestTestCase {
         } else {
             ensureGreen(index); // wait for source index to be available on both nodes before starting shrink
 
-            String updateSettingsRequestBody = "{\"settings\": {\"index.blocks.write\": true}}";
-            Response rsp = client().performRequest("PUT", "/" + index + "/_settings", Collections.emptyMap(),
-                new StringEntity(updateSettingsRequestBody, ContentType.APPLICATION_JSON));
-            assertEquals(200, rsp.getStatusLine().getStatusCode());
+            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
+            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
+            client().performRequest(updateSettingsRequest);
 
-            String shrinkIndexRequestBody = "{\"settings\": {\"index.number_of_shards\": 1}}";
-            rsp = client().performRequest("PUT", "/" + index + "/_shrink/" + shrunkenIndex, Collections.emptyMap(),
-                new StringEntity(shrinkIndexRequestBody, ContentType.APPLICATION_JSON));
-            assertEquals(200, rsp.getStatusLine().getStatusCode());
+            Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex);
+            shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}");
+            client().performRequest(shrinkIndexRequest);
 
             numDocs = countOfIndexedRandomDocuments();
         }
 
-        Response rsp = client().performRequest("POST", "/_refresh");
-        assertEquals(200, rsp.getStatusLine().getStatusCode());
+        client().performRequest(new Request("POST", "/_refresh"));
 
-        Map<?, ?> response = toMap(client().performRequest("GET", "/" + index + "/_search"));
+        Map<?, ?> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
         assertNoFailures(response);
         int totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
         assertThat(totalShards, greaterThan(1));
@@ -486,7 +466,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
         assertEquals(numDocs, totalHits);
 
         if (runningAgainstOldCluster == false) {
-            response = toMap(client().performRequest("GET", "/" + shrunkenIndex + "/_search"));
+            response = entityAsMap(client().performRequest(new Request("GET", "/" + shrunkenIndex + "/_search")));
             assertNoFailures(response);
             totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
             assertEquals(1, totalShards);
@@ -499,43 +479,48 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
     void assertBasicSearchWorks(int count) throws IOException {
         logger.info("--> testing basic search");
-        Map<String, Object> response = toMap(client().performRequest("GET", "/" + index + "/_search"));
-        assertNoFailures(response);
-        int numDocs = (int) XContentMapValues.extractValue("hits.total", response);
-        logger.info("Found {} in old index", numDocs);
-        assertEquals(count, numDocs);
+        {
+            Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
+            assertNoFailures(response);
+            int numDocs = (int) XContentMapValues.extractValue("hits.total", response);
+            logger.info("Found {} in old index", numDocs);
+            assertEquals(count, numDocs);
+        }
 
         logger.info("--> testing basic search with sort");
-        String searchRequestBody = "{ \"sort\": [{ \"int\" : \"asc\" }]}";
-        response = toMap(client().performRequest("GET", "/" + index + "/_search", Collections.emptyMap(),
-            new StringEntity(searchRequestBody, ContentType.APPLICATION_JSON)));
-        assertNoFailures(response);
-        numDocs = (int) XContentMapValues.extractValue("hits.total", response);
-        assertEquals(count, numDocs);
+        {
+            Request searchRequest = new Request("GET", "/" + index + "/_search");
+            searchRequest.setJsonEntity("{ \"sort\": [{ \"int\" : \"asc\" }]}");
+            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
+            assertNoFailures(response);
+            assertTotalHits(count, response);
+        }
 
         logger.info("--> testing exists filter");
-        searchRequestBody = "{ \"query\": { \"exists\" : {\"field\": \"string\"} }}";
-        response = toMap(client().performRequest("GET", "/" + index + "/_search", Collections.emptyMap(),
-            new StringEntity(searchRequestBody, ContentType.APPLICATION_JSON)));
-        assertNoFailures(response);
-        numDocs = (int) XContentMapValues.extractValue("hits.total", response);
-        assertEquals(count, numDocs);
+        {
+            Request searchRequest = new Request("GET", "/" + index + "/_search");
+            searchRequest.setJsonEntity("{ \"query\": { \"exists\" : {\"field\": \"string\"} }}");
+            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
+            assertNoFailures(response);
+            assertTotalHits(count, response);
+        }
 
-        searchRequestBody = "{ \"query\": { \"exists\" : {\"field\": \"field.with.dots\"} }}";
-        response = toMap(client().performRequest("GET", "/" + index + "/_search", Collections.emptyMap(),
-            new StringEntity(searchRequestBody, ContentType.APPLICATION_JSON)));
-        assertNoFailures(response);
-        numDocs = (int) XContentMapValues.extractValue("hits.total", response);
-        assertEquals(count, numDocs);
+        logger.info("--> testing field with dots in the name");
+        {
+            Request searchRequest = new Request("GET", "/" + index + "/_search");
+            searchRequest.setJsonEntity("{ \"query\": { \"exists\" : {\"field\": \"field.with.dots\"} }}");
+            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
+            assertNoFailures(response);
+            assertTotalHits(count, response);
+        }
     }
 
     void assertAllSearchWorks(int count) throws IOException {
         logger.info("--> testing _all search");
-        Map<String, Object>  searchRsp = toMap(client().performRequest("GET", "/" + index + "/_search"));
-        assertNoFailures(searchRsp);
-        int totalHits = (int) XContentMapValues.extractValue("hits.total", searchRsp);
-        assertEquals(count, totalHits);
-        Map<?, ?> bestHit = (Map<?, ?>) ((List<?>)(XContentMapValues.extractValue("hits.hits", searchRsp))).get(0);
+        Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
+        assertNoFailures(response);
+        assertTotalHits(count, response);
+        Map<?, ?> bestHit = (Map<?, ?>) ((List<?>) (XContentMapValues.extractValue("hits.hits", response))).get(0);
 
         // Make sure there are payloads and they are taken into account for the score
         // the 'string' field has a boost of 4 in the mappings so it should get a payload boost
@@ -543,82 +528,77 @@ public class FullClusterRestartIT extends ESRestTestCase {
         assertNotNull(stringValue);
         String type = (String) bestHit.get("_type");
         String id = (String) bestHit.get("_id");
-        String requestBody = "{ \"query\": { \"match_all\" : {} }}";
-        String explanation = toStr(client().performRequest("GET", "/" + index + "/" + type + "/" + id,
-                Collections.emptyMap(), new StringEntity(requestBody, ContentType.APPLICATION_JSON)));
+        Request explanationRequest = new Request("GET", "/" + index + "/" + type + "/" + id + "/_explain");
+        explanationRequest.setJsonEntity("{ \"query\": { \"match_all\" : {} }}");
+        String explanation = toStr(client().performRequest(explanationRequest));
         assertFalse("Could not find payload boost in explanation\n" + explanation, explanation.contains("payloadBoost"));
 
         // Make sure the query can run on the whole index
-        searchRsp = toMap(client().performRequest("GET", "/" + index + "/_search",
-                Collections.singletonMap("explain", "true"), new StringEntity(requestBody, ContentType.APPLICATION_JSON)));
-        assertNoFailures(searchRsp);
-        totalHits = (int) XContentMapValues.extractValue("hits.total", searchRsp);
-        assertEquals(count, totalHits);
+        Request searchRequest = new Request("GET", "/" + index + "/_search");
+        searchRequest.setEntity(explanationRequest.getEntity());
+        searchRequest.addParameter("explain", "true");
+        Map<?, ?> matchAllResponse = entityAsMap(client().performRequest(searchRequest));
+        assertNoFailures(matchAllResponse);
+        assertTotalHits(count, matchAllResponse);
     }
 
     void assertBasicAggregationWorks() throws IOException {
         // histogram on a long
-        String requestBody = "{ \"aggs\": { \"histo\" : {\"histogram\" : {\"field\": \"int\", \"interval\": 10}} }}";
-        Map<?, ?> searchRsp = toMap(client().performRequest("GET", "/" + index + "/_search", Collections.emptyMap(),
-                new StringEntity(requestBody, ContentType.APPLICATION_JSON)));
-        assertNoFailures(searchRsp);
-        List<?> histoBuckets = (List<?>) XContentMapValues.extractValue("aggregations.histo.buckets", searchRsp);
-        long totalCount = 0;
+        Request longHistogramRequest = new Request("GET", "/" + index + "/_search");
+        longHistogramRequest.setJsonEntity("{ \"aggs\": { \"histo\" : {\"histogram\" : {\"field\": \"int\", \"interval\": 10}} }}");
+        Map<?, ?> longHistogram = entityAsMap(client().performRequest(longHistogramRequest));
+        assertNoFailures(longHistogram);
+        List<?> histoBuckets = (List<?>) XContentMapValues.extractValue("aggregations.histo.buckets", longHistogram);
+        int histoCount = 0;
         for (Object entry : histoBuckets) {
             Map<?, ?> bucket = (Map<?, ?>) entry;
-            totalCount += (Integer) bucket.get("doc_count");
+            histoCount += (Integer) bucket.get("doc_count");
         }
-        int totalHits = (int) XContentMapValues.extractValue("hits.total", searchRsp);
-        assertEquals(totalHits, totalCount);
+        assertTotalHits(histoCount, longHistogram);
 
         // terms on a boolean
-        requestBody = "{ \"aggs\": { \"bool_terms\" : {\"terms\" : {\"field\": \"bool\"}} }}";
-        searchRsp = toMap(client().performRequest("GET", "/" + index + "/_search", Collections.emptyMap(),
-                new StringEntity(requestBody, ContentType.APPLICATION_JSON)));
-        List<?> termsBuckets = (List<?>) XContentMapValues.extractValue("aggregations.bool_terms.buckets", searchRsp);
-        totalCount = 0;
+        Request boolTermsRequest = new Request("GET", "/" + index + "/_search");
+        boolTermsRequest.setJsonEntity("{ \"aggs\": { \"bool_terms\" : {\"terms\" : {\"field\": \"bool\"}} }}");
+        Map<?, ?> boolTerms = entityAsMap(client().performRequest(boolTermsRequest));
+        List<?> termsBuckets = (List<?>) XContentMapValues.extractValue("aggregations.bool_terms.buckets", boolTerms);
+        int termsCount = 0;
         for (Object entry : termsBuckets) {
             Map<?, ?> bucket = (Map<?, ?>) entry;
-            totalCount += (Integer) bucket.get("doc_count");
+            termsCount += (Integer) bucket.get("doc_count");
         }
-        totalHits = (int) XContentMapValues.extractValue("hits.total", searchRsp);
-        assertEquals(totalHits, totalCount);
+        assertTotalHits(termsCount, boolTerms);
     }
 
     void assertRealtimeGetWorks() throws IOException {
-        String requestBody = "{ \"index\": { \"refresh_interval\" : -1 }}";
-        Response response = client().performRequest("PUT", "/" + index + "/_settings", Collections.emptyMap(),
-                new StringEntity(requestBody, ContentType.APPLICATION_JSON));
-        assertEquals(200, response.getStatusLine().getStatusCode());
+        Request disableAutoRefresh = new Request("PUT", "/" + index + "/_settings");
+        disableAutoRefresh.setJsonEntity("{ \"index\": { \"refresh_interval\" : -1 }}");
+        client().performRequest(disableAutoRefresh);
 
-        requestBody = "{ \"query\": { \"match_all\" : {} }}";
-        Map<String, Object> searchRsp = toMap(client().performRequest("GET", "/" + index + "/_search", Collections.emptyMap(),
-                new StringEntity(requestBody, ContentType.APPLICATION_JSON)));
-        Map<?, ?> hit = (Map<?, ?>) ((List<?>)(XContentMapValues.extractValue("hits.hits", searchRsp))).get(0);
+        Request searchRequest = new Request("GET", "/" + index + "/_search");
+        searchRequest.setJsonEntity("{ \"query\": { \"match_all\" : {} }}");
+        Map<?, ?> searchResponse = entityAsMap(client().performRequest(searchRequest));
+        Map<?, ?> hit = (Map<?, ?>) ((List<?>)(XContentMapValues.extractValue("hits.hits", searchResponse))).get(0);
         String docId = (String) hit.get("_id");
 
-        requestBody = "{ \"doc\" : { \"foo\": \"bar\"}}";
-        response = client().performRequest("POST", "/" + index + "/doc/" + docId + "/_update", Collections.emptyMap(),
-                new StringEntity(requestBody, ContentType.APPLICATION_JSON));
-        assertEquals(200, response.getStatusLine().getStatusCode());
+        Request updateRequest = new Request("POST", "/" + index + "/doc/" + docId + "/_update");
+        updateRequest.setJsonEntity("{ \"doc\" : { \"foo\": \"bar\"}}");
+        client().performRequest(updateRequest);
 
-        Map<String, Object> getRsp = toMap(client().performRequest("GET", "/" + index + "/doc/" + docId));
+        Map<String, Object> getRsp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/doc/" + docId)));
         Map<?, ?> source = (Map<?, ?>) getRsp.get("_source");
         assertTrue("doc does not contain 'foo' key: " + source, source.containsKey("foo"));
 
-        requestBody = "{ \"index\": { \"refresh_interval\" : \"1s\" }}";
-        response = client().performRequest("PUT", "/" + index + "/_settings", Collections.emptyMap(),
-                new StringEntity(requestBody, ContentType.APPLICATION_JSON));
-        assertEquals(200, response.getStatusLine().getStatusCode());
+        Request enableAutoRefresh = new Request("PUT", "/" + index + "/_settings");
+        enableAutoRefresh.setJsonEntity("{ \"index\": { \"refresh_interval\" : \"1s\" }}");
+        client().performRequest(enableAutoRefresh);
     }
 
     void assertStoredBinaryFields(int count) throws Exception {
-        String requestBody = "{ \"query\": { \"match_all\" : {} }, \"size\": 100, \"stored_fields\": \"binary\"}";
-        Map<String, Object> rsp = toMap(client().performRequest("GET", "/" + index + "/_search",
-            Collections.emptyMap(), new StringEntity(requestBody, ContentType.APPLICATION_JSON)));
+        Request request = new Request("GET", "/" + index + "/_search");
+        request.setJsonEntity("{ \"query\": { \"match_all\" : {} }, \"size\": 100, \"stored_fields\": \"binary\"}");
+        Map<String, Object> rsp = entityAsMap(client().performRequest(request));
 
-        int totalCount = (Integer) XContentMapValues.extractValue("hits.total", rsp);
-        assertEquals(count, totalCount);
+        assertTotalHits(count, rsp);
         List<?> hits = (List<?>) XContentMapValues.extractValue("hits.hits", rsp);
         assertEquals(100, hits.size());
         for (Object hit : hits) {
@@ -631,14 +611,6 @@ public class FullClusterRestartIT extends ESRestTestCase {
         }
     }
 
-    static Map<String, Object> toMap(Response response) throws IOException {
-        return toMap(EntityUtils.toString(response.getEntity()));
-    }
-
-    static Map<String, Object> toMap(String response) throws IOException {
-        return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false);
-    }
-
     static String toStr(Response response) throws IOException {
         return EntityUtils.toString(response.getEntity());
     }
@@ -648,6 +620,11 @@ public class FullClusterRestartIT extends ESRestTestCase {
         assertEquals(0, failed);
     }
 
+    static void assertTotalHits(int expectedTotalHits, Map<?, ?> response) {
+        int actualTotalHits = (Integer) XContentMapValues.extractValue("hits.total", response);
+        assertEquals(expectedTotalHits, actualTotalHits);
+    }
+
     /**
      * Tests that a single document survives. Super basic smoke test.
      */
@@ -656,11 +633,12 @@ public class FullClusterRestartIT extends ESRestTestCase {
         String doc = "{\"test\": \"test\"}";
 
         if (runningAgainstOldCluster) {
-            client().performRequest("PUT", docLocation, singletonMap("refresh", "true"),
-                    new StringEntity(doc, ContentType.APPLICATION_JSON));
+            Request createDoc = new Request("PUT", docLocation);
+            createDoc.setJsonEntity(doc);
+            client().performRequest(createDoc);
         }
 
-        assertThat(toStr(client().performRequest("GET", docLocation)), containsString(doc));
+        assertThat(toStr(client().performRequest(new Request("GET", docLocation))), containsString(doc));
     }
 
     /**
@@ -733,16 +711,18 @@ public class FullClusterRestartIT extends ESRestTestCase {
         }
 
         // Count the documents in the index to make sure we have as many as we put there
-        String countResponse = toStr(client().performRequest("GET", "/" + index + "/_search", singletonMap("size", "0")));
+        Request countRequest = new Request("GET", "/" + index + "/_search");
+        countRequest.addParameter("size", "0");
+        String countResponse = toStr(client().performRequest(countRequest));
         assertThat(countResponse, containsString("\"total\":" + count));
 
         if (false == runningAgainstOldCluster) {
             boolean restoredFromTranslog = false;
             boolean foundPrimary = false;
-            Map<String, String> params = new HashMap<>();
-            params.put("h", "index,shard,type,stage,translog_ops_recovered");
-            params.put("s", "index,shard,type");
-            String recoveryResponse = toStr(client().performRequest("GET", "/_cat/recovery/" + index, params));
+            Request recoveryRequest = new Request("GET", "/_cat/recovery/" + index);
+            recoveryRequest.addParameter("h", "index,shard,type,stage,translog_ops_recovered");
+            recoveryRequest.addParameter("s", "index,shard,type");
+            String recoveryResponse = toStr(client().performRequest(recoveryRequest));
             for (String line : recoveryResponse.split("\n")) {
                 // Find the primaries
                 foundPrimary = true;
@@ -768,11 +748,10 @@ public class FullClusterRestartIT extends ESRestTestCase {
         if (shouldHaveTranslog && false == currentLuceneVersion.equals(bwcLuceneVersion)) {
             int numCurrentVersion = 0;
             int numBwcVersion = 0;
-            params.clear();
-            params.put("h", "prirep,shard,index,version");
-            params.put("s", "prirep,shard,index");
-            String segmentsResponse = toStr(
-                    client().performRequest("GET", "/_cat/segments/" + index, params));
+            Request segmentsRequest = new Request("GET", "/_cat/segments/" + index);
+            segmentsRequest.addParameter("h", "prirep,shard,index,version");
+            segmentsRequest.addParameter("s", "prirep,shard,index");
+            String segmentsResponse = toStr(client().performRequest(segmentsRequest));
             for (String line : segmentsResponse.split("\n")) {
                 if (false == line.startsWith("p")) {
                     continue;
@@ -817,14 +796,16 @@ public class FullClusterRestartIT extends ESRestTestCase {
         refresh();
 
         // Count the documents in the index to make sure we have as many as we put there
-        String countResponse = toStr(client().performRequest("GET", "/" + index + "/_search", singletonMap("size", "0")));
+        Request countRequest = new Request("GET", "/" + index + "/_search");
+        countRequest.addParameter("size", "0");
+        String countResponse = toStr(client().performRequest(countRequest));
         assertThat(countResponse, containsString("\"total\":" + count));
 
         // Stick a routing attribute into to cluster settings so we can see it after the restore
-        HttpEntity routingSetting = new StringEntity(
-                "{\"persistent\": {\"cluster.routing.allocation.exclude.test_attr\": \"" + oldClusterVersion + "\"}}",
-                ContentType.APPLICATION_JSON);
-        client().performRequest("PUT", "/_cluster/settings", emptyMap(), routingSetting);
+        Request addRoutingSettings = new Request("PUT", "/_cluster/settings");
+        addRoutingSettings.setJsonEntity(
+                    "{\"persistent\": {\"cluster.routing.allocation.exclude.test_attr\": \"" + oldClusterVersion + "\"}}");
+        client().performRequest(addRoutingSettings);
 
         // Stick a template into the cluster so we can see it after the restore
         XContentBuilder templateBuilder = JsonXContent.contentBuilder().startObject();
@@ -857,8 +838,9 @@ public class FullClusterRestartIT extends ESRestTestCase {
             templateBuilder.endObject();
         }
         templateBuilder.endObject().endObject();
-        client().performRequest("PUT", "/_template/test_template", emptyMap(),
-                new StringEntity(Strings.toString(templateBuilder), ContentType.APPLICATION_JSON));
+        Request createTemplateRequest = new Request("PUT", "/_template/test_template");
+        createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder));
+        client().performRequest(createTemplateRequest);
 
         if (runningAgainstOldCluster) {
             // Create the repo
@@ -871,13 +853,15 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 repoConfig.endObject();
             }
             repoConfig.endObject();
-            client().performRequest("PUT", "/_snapshot/repo", emptyMap(),
-                    new StringEntity(Strings.toString(repoConfig), ContentType.APPLICATION_JSON));
+            Request createRepoRequest = new Request("PUT", "/_snapshot/repo");
+            createRepoRequest.setJsonEntity(Strings.toString(repoConfig));
+            client().performRequest(createRepoRequest);
         }
 
-        client().performRequest("PUT", "/_snapshot/repo/" + (runningAgainstOldCluster ? "old_snap" : "new_snap"),
-                singletonMap("wait_for_completion", "true"),
-                new StringEntity("{\"indices\": \"" + index + "\"}", ContentType.APPLICATION_JSON));
+        Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + (runningAgainstOldCluster ? "old_snap" : "new_snap"));
+        createSnapshot.addParameter("wait_for_completion", "true");
+        createSnapshot.setJsonEntity("{\"indices\": \"" + index + "\"}");
+        client().performRequest(createSnapshot);
 
         checkSnapshot("old_snap", count, oldClusterVersion);
         if (false == runningAgainstOldCluster) {
@@ -896,10 +880,13 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
-            client().performRequest("PUT", "/" + index, Collections.emptyMap(),
-                new StringEntity(Strings.toString(mappingsAndSettings), ContentType.APPLICATION_JSON));
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
         } else {
-            Response response = client().performRequest("GET", index + "/_stats", singletonMap("level", "shards"));
+            Request statsRequest = new Request("GET", index + "/_stats");
+            statsRequest.addParameter("level", "shards");
+            Response response = client().performRequest(statsRequest);
             List<Object> shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + index + ".shards.0");
             String globalHistoryUUID = null;
             for (Object shard : shardStats) {
@@ -920,18 +907,20 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
     private void checkSnapshot(String snapshotName, int count, Version tookOnVersion) throws IOException {
         // Check the snapshot metadata, especially the version
-        String response = toStr(client().performRequest("GET", "/_snapshot/repo/" + snapshotName, listSnapshotVerboseParams()));
-        Map<String, Object> map = toMap(response);
-        assertEquals(response, singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", map));
-        assertEquals(response, singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", map));
-        assertEquals(response, singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", map));
+        Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName);
+        if (false == (runningAgainstOldCluster && oldClusterVersion.before(Version.V_5_5_0))) {
+            listSnapshotRequest.addParameter("verbose", "true");
+        }
+        Map<String, Object> listSnapshotResponse = entityAsMap(client().performRequest(listSnapshotRequest));
+        assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", listSnapshotResponse));
+        assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", listSnapshotResponse));
+        assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", listSnapshotResponse));
 
         // Remove the routing setting and template so we can test restoring them.
-        HttpEntity clearRoutingSetting = new StringEntity(
-                "{\"persistent\":{\"cluster.routing.allocation.exclude.test_attr\": null}}",
-                ContentType.APPLICATION_JSON);
-        client().performRequest("PUT", "/_cluster/settings", emptyMap(), clearRoutingSetting);
-        client().performRequest("DELETE", "/_template/test_template", emptyMap(), clearRoutingSetting);
+        Request clearRoutingFromSettings = new Request("PUT", "/_cluster/settings");
+        clearRoutingFromSettings.setJsonEntity("{\"persistent\":{\"cluster.routing.allocation.exclude.test_attr\": null}}");
+        client().performRequest(clearRoutingFromSettings);
+        client().performRequest(new Request("DELETE", "/_template/test_template"));
 
         // Restore
         XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject();
@@ -940,11 +929,15 @@ public class FullClusterRestartIT extends ESRestTestCase {
         restoreCommand.field("rename_pattern", index);
         restoreCommand.field("rename_replacement", "restored_" + index);
         restoreCommand.endObject();
-        client().performRequest("POST", "/_snapshot/repo/" + snapshotName + "/_restore", singletonMap("wait_for_completion", "true"),
-                new StringEntity(Strings.toString(restoreCommand), ContentType.APPLICATION_JSON));
+        Request restoreRequest = new Request("POST", "/_snapshot/repo/" + snapshotName + "/_restore");
+        restoreRequest.addParameter("wait_for_completion", "true");
+        restoreRequest.setJsonEntity(Strings.toString(restoreCommand));
+        client().performRequest(restoreRequest);
 
         // Make sure search finds all documents
-        String countResponse = toStr(client().performRequest("GET", "/restored_" + index + "/_search", singletonMap("size", "0")));
+        Request countRequest = new Request("GET", "/restored_" + index + "/_search");
+        countRequest.addParameter("size", "0");
+        String countResponse = toStr(client().performRequest(countRequest));
         assertThat(countResponse, containsString("\"total\":" + count));
 
         // Add some extra documents to the index to be sure we can still write to it after restoring it
@@ -954,61 +947,56 @@ public class FullClusterRestartIT extends ESRestTestCase {
             bulk.append("{\"index\":{\"_id\":\"").append(count + i).append("\"}}\n");
             bulk.append("{\"test\":\"test\"}\n");
         }
-        client().performRequest("POST", "/restored_" + index + "/doc/_bulk", singletonMap("refresh", "true"),
-                new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON));
+        Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/doc/_bulk");
+        writeToRestoredRequest.addParameter("refresh", "true");
+        writeToRestoredRequest.setJsonEntity(bulk.toString());
+        client().performRequest(writeToRestoredRequest);
 
         // And count to make sure the add worked
         // Make sure search finds all documents
-        countResponse = toStr(client().performRequest("GET", "/restored_" + index + "/_search", singletonMap("size", "0")));
-        assertThat(countResponse, containsString("\"total\":" + (count + extras)));
+        Request countAfterWriteRequest = new Request("GET", "/restored_" + index + "/_search");
+        countAfterWriteRequest.addParameter("size", "0");
+        String countAfterWriteResponse = toStr(client().performRequest(countAfterWriteRequest));
+        assertThat(countAfterWriteResponse, containsString("\"total\":" + (count + extras)));
 
         // Clean up the index for the next iteration
-        client().performRequest("DELETE", "/restored_*");
+        client().performRequest(new Request("DELETE", "/restored_*"));
 
         // Check settings added by the restore process
-        map = toMap(client().performRequest("GET", "/_cluster/settings", singletonMap("flat_settings", "true")));
-        Map<String, Object> expected = new HashMap<>();
-        expected.put("transient", emptyMap());
-        expected.put("persistent", singletonMap("cluster.routing.allocation.exclude.test_attr", oldClusterVersion.toString()));
-        if (expected.equals(map) == false) {
+        Request clusterSettingsRequest = new Request("GET", "/_cluster/settings");
+        clusterSettingsRequest.addParameter("flat_settings", "true");
+        Map<String, Object> clusterSettingsResponse = entityAsMap(client().performRequest(clusterSettingsRequest));
+        Map<String, Object> expectedClusterSettings = new HashMap<>();
+        expectedClusterSettings.put("transient", emptyMap());
+        expectedClusterSettings.put("persistent",
+                singletonMap("cluster.routing.allocation.exclude.test_attr", oldClusterVersion.toString()));
+        if (expectedClusterSettings.equals(clusterSettingsResponse) == false) {
             NotEqualMessageBuilder builder = new NotEqualMessageBuilder();
-            builder.compareMaps(map, expected);
+            builder.compareMaps(clusterSettingsResponse, expectedClusterSettings);
             fail("settings don't match:\n" + builder.toString());
         }
 
         // Check that the template was restored successfully
-        map = toMap(client().performRequest("GET", "/_template/test_template"));
-        expected = new HashMap<>();
+        Map<String, Object> getTemplateResponse = entityAsMap(client().performRequest(new Request("GET", "/_template/test_template")));
+        Map<String, Object> expectedTemplate = new HashMap<>();
         if (runningAgainstOldCluster && oldClusterVersion.before(Version.V_6_0_0_beta1)) {
-            expected.put("template", "evil_*");
+            expectedTemplate.put("template", "evil_*");
         } else {
-            expected.put("index_patterns", singletonList("evil_*"));
+            expectedTemplate.put("index_patterns", singletonList("evil_*"));
         }
-        expected.put("settings", singletonMap("index", singletonMap("number_of_shards", "1")));
-        expected.put("mappings", singletonMap("doc", singletonMap("_source", singletonMap("enabled", true))));
-        expected.put("order", 0);
+        expectedTemplate.put("settings", singletonMap("index", singletonMap("number_of_shards", "1")));
+        expectedTemplate.put("mappings", singletonMap("doc", singletonMap("_source", singletonMap("enabled", true))));
+        expectedTemplate.put("order", 0);
         Map<String, Object> aliases = new HashMap<>();
         aliases.put("alias1", emptyMap());
         aliases.put("alias2", singletonMap("filter", singletonMap("term", singletonMap("version", tookOnVersion.toString()))));
-        expected.put("aliases", aliases);
-        expected = singletonMap("test_template", expected);
-        if (false == expected.equals(map)) {
+        expectedTemplate.put("aliases", aliases);
+        expectedTemplate = singletonMap("test_template", expectedTemplate);
+        if (false == expectedTemplate.equals(getTemplateResponse)) {
             NotEqualMessageBuilder builder = new NotEqualMessageBuilder();
-            builder.compareMaps(map, expected);
+            builder.compareMaps(getTemplateResponse, expectedTemplate);
             fail("template doesn't match:\n" + builder.toString());
         }
-
-    }
-
-    /**
-     * Parameters required to get the version of Elasticsearch that took the snapshot.
-     * On versions after 5.5 we need a {@code verbose} parameter.
-     */
-    private Map<String, String> listSnapshotVerboseParams() {
-        if (runningAgainstOldCluster && oldClusterVersion.before(Version.V_5_5_0)) {
-            return emptyMap();
-        }
-        return singletonMap("verbose", "true");
     }
 
     // TODO tests for upgrades after shrink. We've had trouble with shrink in the past.
@@ -1018,14 +1006,15 @@ public class FullClusterRestartIT extends ESRestTestCase {
         logger.info("Indexing {} random documents", count);
         for (int i = 0; i < count; i++) {
             logger.debug("Indexing document [{}]", i);
-            client().performRequest("POST", "/" + index + "/doc/" + i, emptyMap(),
-                    new StringEntity(Strings.toString(docSupplier.apply(i)), ContentType.APPLICATION_JSON));
+            Request createDocument = new Request("POST", "/" + index + "/doc/" + i);
+            createDocument.setJsonEntity(Strings.toString(docSupplier.apply(i)));
+            client().performRequest(createDocument);
             if (rarely()) {
                 refresh();
             }
             if (flushAllowed && rarely()) {
                 logger.debug("Flushing [{}]", index);
-                client().performRequest("POST", "/" + index + "/_flush");
+                client().performRequest(new Request("POST", "/" + index + "/_flush"));
             }
         }
         if (saveInfo) {
@@ -1042,13 +1031,16 @@ public class FullClusterRestartIT extends ESRestTestCase {
         infoDoc.field("value", value);
         infoDoc.endObject();
         // Only create the first version so we know how many documents are created when the index is first created
-        Map<String, String> params = singletonMap("op_type", "create");
-        client().performRequest("PUT", "/info/doc/" + index + "_" + type, params,
-                new StringEntity(Strings.toString(infoDoc), ContentType.APPLICATION_JSON));
+        Request request = new Request("PUT", "/info/doc/" + index + "_" + type);
+        request.addParameter("op_type", "create");
+        request.setJsonEntity(Strings.toString(infoDoc));
+        client().performRequest(request);
     }
 
     private String loadInfoDocument(String type) throws IOException {
-        String doc = toStr(client().performRequest("GET", "/info/doc/" + index + "_" + type, singletonMap("filter_path", "_source")));
+        Request request = new Request("GET", "/info/doc/" + index + "_" + type);
+        request.addParameter("filter_path", "_source");
+        String doc = toStr(client().performRequest(request));
         Matcher m = Pattern.compile("\"value\":\"(.+)\"").matcher(doc);
         assertTrue(doc, m.find());
         return m.group(1);
@@ -1060,11 +1052,13 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
     private void refresh() throws IOException {
         logger.debug("Refreshing [{}]", index);
-        client().performRequest("POST", "/" + index + "/_refresh");
+        client().performRequest(new Request("POST", "/" + index + "/_refresh"));
     }
 
     private List<String> dataNodes(String index, RestClient client) throws IOException {
-        Response response = client.performRequest("GET", index + "/_stats", singletonMap("level", "shards"));
+        Request request = new Request("GET", index + "/_stats");
+        request.addParameter("level", "shards");
+        Response response = client.performRequest(request);
         List<String> nodes = new ArrayList<>();
         List<Object> shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + index + ".shards.0");
         for (Object shard : shardStats) {
@@ -1073,4 +1067,21 @@ public class FullClusterRestartIT extends ESRestTestCase {
         }
         return nodes;
     }
+
+    /**
+     * Wait for an index to have green health, waiting longer than
+     * {@link ESRestTestCase#ensureGreen}.
+     */
+    protected void ensureGreenLongWait(String index) throws IOException {
+        Request request = new Request("GET", "/_cluster/health/" + index);
+        request.addParameter("timeout", "2m");
+        request.addParameter("wait_for_status", "green");
+        request.addParameter("wait_for_no_relocating_shards", "true");
+        request.addParameter("wait_for_events", "languid");
+        request.addParameter("level", "shards");
+        Map<String, Object> healthRsp = entityAsMap(client().performRequest(request));
+        logger.info("health api response: {}", healthRsp);
+        assertEquals("green", healthRsp.get("status"));
+        assertFalse((Boolean) healthRsp.get("timed_out"));
+    }
 }

From 1390a849e10568b40a5333059aebfe94f0239411 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Sat, 21 Jul 2018 10:07:08 +0200
Subject: [PATCH 129/260] [TEST] improve REST high-level client naming
 conventions check (#32244)

Check the deprecated methods are effectively deprecated.
Also compare the class rather than their names when checking argument types.
---
 .../client/RestHighLevelClientTests.java      | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 5acc6f5552f..64a344790ca 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -724,8 +724,8 @@ public class RestHighLevelClientTests extends ESTestCase {
                 assertEquals(0, method.getExceptionTypes().length);
                 assertEquals(3, method.getParameterTypes().length);
                 assertThat(method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
-                assertThat(method.getParameterTypes()[1].getName(), equalTo(RequestOptions.class.getName()));
-                assertThat(method.getParameterTypes()[2].getName(), equalTo(ActionListener.class.getName()));
+                assertThat(method.getParameterTypes()[1], equalTo(RequestOptions.class));
+                assertThat(method.getParameterTypes()[2], equalTo(ActionListener.class));
             } else {
                 //A few methods return a boolean rather than a response object
                 if (apiName.equals("ping") || apiName.contains("exist")) {
@@ -738,18 +738,23 @@ public class RestHighLevelClientTests extends ESTestCase {
                 //a few methods don't accept a request object as argument
                 if (apiName.equals("ping") || apiName.equals("info")) {
                     assertEquals(1, method.getParameterTypes().length);
-                    assertThat(method.getParameterTypes()[0].getName(), equalTo(RequestOptions.class.getName()));
+                    assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class));
                 } else {
                     assertEquals(apiName, 2, method.getParameterTypes().length);
                     assertThat(method.getParameterTypes()[0].getSimpleName(), endsWith("Request"));
-                    assertThat(method.getParameterTypes()[1].getName(), equalTo(RequestOptions.class.getName()));
+                    assertThat(method.getParameterTypes()[1], equalTo(RequestOptions.class));
                 }
 
                 boolean remove = apiSpec.remove(apiName);
-                if (remove == false && deprecatedMethods.contains(apiName) == false) {
-                    //TODO xpack api are currently ignored, we need to load xpack yaml spec too
-                    if (apiName.startsWith("xpack.") == false) {
-                        apiNotFound.add(apiName);
+                if (remove == false) {
+                    if (deprecatedMethods.contains(apiName)) {
+                        assertTrue("method [" + method.getName() + "], api [" + apiName + "] should be deprecated",
+                            method.isAnnotationPresent(Deprecated.class));
+                    } else {
+                        //TODO xpack api are currently ignored, we need to load xpack yaml spec too
+                        if (apiName.startsWith("xpack.") == false) {
+                            apiNotFound.add(apiName);
+                        }
                     }
                 }
             }

From 0a511cc76f770f910e512cbbe76ab27f966e1a26 Mon Sep 17 00:00:00 2001
From: Alpar Torok <torokalpar@gmail.com>
Date: Sat, 21 Jul 2018 12:50:17 +0000
Subject: [PATCH 130/260] Improve message when JAVA_HOME not set (#32022)

closes #31399
---
 .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 9f54ae8b682..5da5912dabe 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -222,7 +222,11 @@ class BuildPlugin implements Plugin<Project> {
                 // IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
                 return Jvm.current().javaHome
             } else {
-                throw new GradleException("JAVA_HOME must be set to build Elasticsearch")
+                throw new GradleException(
+                        "JAVA_HOME must be set to build Elasticsearch. " +
+                                "Note that if the variable was just set you might have to run `./gradlew --stop` for " +
+                                "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
+                )
             }
         }
         return javaHome

From 8f1d15ce61bd9b2c7f4025eaff8d1848cc282a9c Mon Sep 17 00:00:00 2001
From: Nicolas Ruflin <spam@ruflin.com>
Date: Mon, 23 Jul 2018 07:38:19 +0200
Subject: [PATCH 131/260] Add new fields to monitoring template for Beats state
 (#32085)

New data is reported from Beats to the monitoring endpoint. This PR adds the template change necessary for it. See https://github.com/elastic/beats/issues/7521 for more details.

Queue data is skipped for now as implementation is not finished yet.
---
 .../src/main/resources/monitoring-beats.json  | 63 +++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/x-pack/plugin/core/src/main/resources/monitoring-beats.json b/x-pack/plugin/core/src/main/resources/monitoring-beats.json
index ed027387a49..07756ba2602 100644
--- a/x-pack/plugin/core/src/main/resources/monitoring-beats.json
+++ b/x-pack/plugin/core/src/main/resources/monitoring-beats.json
@@ -37,6 +37,49 @@
             },
             "state": {
               "properties": {
+                "beat": {
+                  "properties": {
+                    "name": {
+                      "type": "keyword"
+                    }
+                  }
+                },
+                "host": {
+                  "properties": {
+                    "architecture": {
+                      "type": "keyword"
+                    },
+                    "name": {
+                      "type": "keyword"
+                    },
+                    "os": {
+                      "properties": {
+                        "build": {
+                          "type": "keyword"
+                        },
+                        "family": {
+                          "type": "keyword"
+                        },
+                        "platform": {
+                          "type": "keyword"
+                        },
+                        "version": {
+                          "type": "keyword"
+                        }
+                      }
+                    }
+                  }
+                },
+                "input": {
+                  "properties": {
+                    "count": {
+                      "type": "long"
+                    },
+                    "names": {
+                      "type": "keyword"
+                    }
+                  }
+                },
                 "module": {
                   "properties": {
                     "count": {
@@ -46,6 +89,26 @@
                       "type": "keyword"
                     }
                   }
+                },
+                "output": {
+                  "properties": {
+                    "name": {
+                      "type": "keyword"
+                    }
+                  }
+                },
+                "service": {
+                  "properties": {
+                    "id": {
+                      "type": "keyword"
+                    },
+                    "name": {
+                      "type": "keyword"
+                    },
+                    "version": {
+                      "type": "keyword"
+                    }
+                  }
                 }
               }
             },

From 4b3284f7cbe679b94f10178468566fb0835d8a73 Mon Sep 17 00:00:00 2001
From: itsnotv <itsnotv@users.noreply.github.com>
Date: Mon, 23 Jul 2018 01:09:52 -0700
Subject: [PATCH 132/260] CCE when re-throwing "shard not available" exception
 in TransportShardMultiGetAction (#32185)

ClassCastException can be thrown by callers of TransportActions.isShardNotAvailableException(e) as e is not always an instance of ElasticSearchException

fixes #32173
---
 .../action/get/TransportShardMultiGetAction.java      |  5 ++---
 .../TransportShardMultiTermsVectorAction.java         | 11 +++++------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index d15b7b92d62..e0a6cd82786 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.get;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -90,9 +89,9 @@ public class TransportShardMultiGetAction extends TransportSingleShardAction<Mul
                 GetResult getResult = indexShard.getService().get(item.type(), item.id(), item.storedFields(), request.realtime(), item.version(),
                     item.versionType(), item.fetchSourceContext());
                 response.add(request.locations.get(i), new GetResponse(getResult));
-            } catch (Exception e) {
+            } catch (RuntimeException e) {
                 if (TransportActions.isShardNotAvailableException(e)) {
-                    throw (ElasticsearchException) e;
+                    throw e;
                 } else {
                     logger.debug(() -> new ParameterizedMessage("{} failed to execute multi_get for [{}]/[{}]", shardId,
                         item.type(), item.id()), e);
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
index b83ac3881fd..f1641fdd25c 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.termvectors;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -84,13 +83,13 @@ public class TransportShardMultiTermsVectorAction extends TransportSingleShardAc
             try {
                 TermVectorsResponse termVectorsResponse = TermVectorsService.getTermVectors(indexShard, termVectorsRequest);
                 response.add(request.locations.get(i), termVectorsResponse);
-            } catch (Exception t) {
-                if (TransportActions.isShardNotAvailableException(t)) {
-                    throw (ElasticsearchException) t;
+            } catch (RuntimeException e) {
+                if (TransportActions.isShardNotAvailableException(e)) {
+                    throw e;
                 } else {
-                    logger.debug(() -> new ParameterizedMessage("{} failed to execute multi term vectors for [{}]/[{}]", shardId, termVectorsRequest.type(), termVectorsRequest.id()), t);
+                    logger.debug(() -> new ParameterizedMessage("{} failed to execute multi term vectors for [{}]/[{}]", shardId, termVectorsRequest.type(), termVectorsRequest.id()), e);
                     response.add(request.locations.get(i),
-                            new MultiTermVectorsResponse.Failure(request.index(), termVectorsRequest.type(), termVectorsRequest.id(), t));
+                            new MultiTermVectorsResponse.Failure(request.index(), termVectorsRequest.type(), termVectorsRequest.id(), e));
                 }
             }
         }

From ff87b7aba42926dea95f65933a7625636d718c87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Mon, 23 Jul 2018 11:31:04 +0200
Subject: [PATCH 133/260] Remove unnecessary warning supressions (#32250)

---
 .../client/PreBuiltTransportClient.java       |  1 +
 .../nio/ChannelFactoryTests.java              |  1 -
 .../elasticsearch/nio/EventHandlerTests.java  |  1 -
 .../nio/SocketChannelContextTests.java        |  1 -
 .../matrix/stats/MultiPassStats.java          |  1 -
 .../ingest/common/JsonProcessorTests.java     |  1 -
 .../painless/InitializerTests.java            |  6 ++---
 .../reindex/RestUpdateByQueryAction.java      |  1 -
 .../netty4/SimpleNetty4TransportTests.java    |  1 -
 .../http/nio/HttpReadWriteHandlerTests.java   |  2 +-
 .../nio/SimpleNioTransportTests.java          |  1 -
 .../settings/ClusterGetSettingsResponse.java  |  1 -
 .../validate/query/QueryExplanation.java      |  1 -
 .../elasticsearch/action/get/GetResponse.java |  1 -
 .../ingest/SimulateProcessorResult.java       |  4 +--
 .../ingest/WriteableIngestDocument.java       |  1 -
 .../action/support/ActiveShardCount.java      |  3 ++-
 .../ReplicationRequestBuilder.java            |  1 -
 .../cluster/RestoreInProgress.java            |  5 ++--
 .../cluster/metadata/IndexGraveyard.java      |  5 ++--
 .../cluster/metadata/IndexMetaData.java       |  2 +-
 .../metadata/IndexTemplateMetaData.java       |  1 -
 .../cluster/metadata/MetaData.java            |  4 +--
 .../cluster/routing/RecoverySource.java       |  2 +-
 .../AbstractAllocationDecision.java           |  2 +-
 .../AllocateUnassignedDecision.java           |  2 +-
 .../routing/allocation/MoveDecision.java      |  2 +-
 .../component/AbstractLifecycleComponent.java |  2 --
 .../inject/ConstructorInjectorStore.java      |  1 -
 .../inject/TypeConverterBindingProcessor.java |  2 --
 .../assistedinject/AssistedConstructor.java   |  1 -
 .../inject/internal/ProviderMethod.java       |  2 +-
 .../inject/multibindings/Multibinder.java     |  1 -
 .../spi/DefaultBindingTargetVisitor.java      |  3 +--
 .../common/io/stream/StreamInput.java         |  1 -
 .../elasticsearch/index/get/GetResult.java    |  1 -
 .../index/mapper/DocumentMapper.java          |  1 -
 .../NodePersistentTasksExecutor.java          |  1 -
 .../PersistentTasksClusterService.java        |  1 -
 .../PersistentTasksExecutorRegistry.java      |  1 -
 .../elasticsearch/repositories/IndexId.java   |  2 +-
 .../repositories/RepositoryData.java          |  2 +-
 .../bucket/composite/InternalComposite.java   |  1 -
 .../bucket/range/RangeAggregator.java         |  2 --
 .../MovFnPipelineAggregationBuilder.java      |  1 -
 .../support/ValuesSourceConfig.java           |  1 -
 .../search/sort/GeoDistanceSortBuilder.java   |  1 -
 .../org/elasticsearch/snapshots/Snapshot.java |  2 +-
 .../elasticsearch/snapshots/SnapshotId.java   |  3 +--
 .../elasticsearch/threadpool/ThreadPool.java  |  6 +++--
 .../transport/TransportResponseHandler.java   |  1 -
 .../indices/stats/IndicesStatsTests.java      |  1 -
 .../cluster/metadata/MetaDataTests.java       |  1 -
 .../blobstore/BlobStoreRepositoryTests.java   |  4 +--
 .../aggregations/bucket/DoubleTermsIT.java    |  3 +--
 .../aggregations/bucket/LongTermsIT.java      |  1 -
 .../aggregations/bucket/MinDocCountIT.java    |  2 +-
 .../search/aggregations/bucket/RangeIT.java   |  1 -
 .../composite/CompositeAggregatorTests.java   |  1 -
 .../bucket/terms/StringTermsIT.java           |  4 ---
 .../CumulativeSumAggregatorTests.java         |  2 --
 .../pipeline/movfn/MovFnUnitTests.java        |  1 -
 .../search/fields/SearchFieldsIT.java         |  3 ---
 .../search/functionscore/FunctionScoreIT.java |  4 +--
 .../functionscore/RandomScoreFunctionIT.java  |  2 --
 .../search/sort/FieldSortIT.java              |  3 ---
 .../search/sort/SimpleSortIT.java             |  4 +--
 .../ESBlobStoreRepositoryIntegTestCase.java   |  2 +-
 .../org/elasticsearch/test/ESTestCase.java    |  3 +--
 .../hamcrest/ElasticsearchAssertions.java     |  3 ++-
 .../yaml/section/GreaterThanAssertion.java    |  1 -
 .../transport/MockTcpTransportTests.java      |  2 +-
 .../nio/SimpleMockNioTransportTests.java      |  1 -
 .../core/security/client/SecurityClient.java  |  3 ---
 .../upgrade/actions/IndexUpgradeAction.java   |  1 -
 .../integration/MlRestTestStateCleaner.java   |  1 -
 .../ml/job/results/AnomalyRecordTests.java    |  3 +--
 .../rollup/RollupRestTestStateCleaner.java    |  1 -
 .../ml/job/persistence/JobProviderTests.java  | 26 +++++++++----------
 .../persistence/JobResultsPersisterTests.java |  2 +-
 .../normalizer/ScoresUpdaterTests.java        |  1 -
 .../monitoring/integration/MonitoringIT.java  |  8 ++----
 .../security/authc/AuthenticationService.java |  1 -
 .../action/RestAuthenticateActionTests.java   |  1 -
 .../nio/SimpleSecurityNioTransportTests.java  |  8 +++---
 .../xpack/test/rest/XPackRestIT.java          |  2 +-
 .../ActiveDirectorySessionFactoryTests.java   |  2 +-
 87 files changed, 64 insertions(+), 139 deletions(-)

diff --git a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
index 7f9bcc6ea08..7bde7fbc06f 100644
--- a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
+++ b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
@@ -21,6 +21,7 @@ package org.elasticsearch.transport.client;
 
 import io.netty.util.ThreadDeathWatcher;
 import io.netty.util.concurrent.GlobalEventExecutor;
+
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.NetworkModule;
diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/ChannelFactoryTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/ChannelFactoryTests.java
index 8ff0cfcd0c8..af4eabefd94 100644
--- a/libs/nio/src/test/java/org/elasticsearch/nio/ChannelFactoryTests.java
+++ b/libs/nio/src/test/java/org/elasticsearch/nio/ChannelFactoryTests.java
@@ -137,7 +137,6 @@ public class ChannelFactoryTests extends ESTestCase {
             super(rawChannelFactory);
         }
 
-        @SuppressWarnings("unchecked")
         @Override
         public NioSocketChannel createChannel(NioSelector selector, SocketChannel channel) throws IOException {
             NioSocketChannel nioSocketChannel = new NioSocketChannel(channel);
diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/EventHandlerTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/EventHandlerTests.java
index 0cc3aa04800..6e1e34ec1f5 100644
--- a/libs/nio/src/test/java/org/elasticsearch/nio/EventHandlerTests.java
+++ b/libs/nio/src/test/java/org/elasticsearch/nio/EventHandlerTests.java
@@ -120,7 +120,6 @@ public class EventHandlerTests extends ESTestCase {
         verify(channelFactory, times(2)).acceptNioChannel(same(serverContext), same(selectorSupplier));
     }
 
-    @SuppressWarnings("unchecked")
     public void testHandleAcceptCallsServerAcceptCallback() throws IOException {
         NioSocketChannel childChannel = new NioSocketChannel(mock(SocketChannel.class));
         SocketChannelContext childContext = mock(SocketChannelContext.class);
diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java
index bc9a7c33f0f..9dbf483107b 100644
--- a/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java
+++ b/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java
@@ -275,7 +275,6 @@ public class SocketChannelContextTests extends ESTestCase {
         }
     }
 
-    @SuppressWarnings("unchecked")
     public void testCloseClosesChannelBuffer() throws IOException {
         try (SocketChannel realChannel = SocketChannel.open()) {
             when(channel.getRawChannel()).thenReturn(realChannel);
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java
index 70e2172ce92..b5a348f45eb 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java
@@ -43,7 +43,6 @@ class MultiPassStats {
         this.fieldBKey = fieldBName;
     }
 
-    @SuppressWarnings("unchecked")
     void computeStats(final List<Double> fieldA, final List<Double> fieldB) {
         // set count
         count = fieldA.size();
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
index 2867ed1d240..099e8e1866b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -146,7 +146,6 @@ public class JsonProcessorTests extends ESTestCase {
         assertThat(exception.getMessage(), equalTo("field [field] not present as part of path [field]"));
     }
 
-    @SuppressWarnings("unchecked")
     public void testAddToRoot() throws Exception {
         String processorTag = randomAlphaOfLength(3);
         String randomTargetField = randomAlphaOfLength(2);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java
index 5d881632dee..d0d0b2165ca 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java
@@ -26,7 +26,7 @@ import java.util.Map;
 
 public class InitializerTests extends ScriptTestCase {
 
-    @SuppressWarnings({"unchecked", "rawtypes"})
+    @SuppressWarnings({"rawtypes"})
     public void testArrayInitializers() {
         int[] ints = (int[])exec("new int[] {}");
 
@@ -59,7 +59,7 @@ public class InitializerTests extends ScriptTestCase {
         assertEquals("aaaaaa", objects[3]);
     }
 
-    @SuppressWarnings({"unchecked", "rawtypes"})
+    @SuppressWarnings({"rawtypes"})
     public void testListInitializers() {
         List list = (List)exec("[]");
 
@@ -91,7 +91,7 @@ public class InitializerTests extends ScriptTestCase {
         assertEquals("aaaaaa", list.get(3));
     }
 
-    @SuppressWarnings({"unchecked", "rawtypes"})
+    @SuppressWarnings({"rawtypes"})
     public void testMapInitializers() {
         Map map = (Map)exec("[:]");
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index 8f09afbb17c..bf0adc6e142 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -57,7 +57,6 @@ public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<Upda
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     protected UpdateByQueryRequest buildRequest(RestRequest request) throws IOException {
         /*
          * Passing the search request through UpdateByQueryRequest first allows
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
index 760ac1253c6..8d628ace2ee 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
@@ -90,7 +90,6 @@ public class SimpleNetty4TransportTests extends AbstractSimpleTransportTestCase
     @Override
     protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
         final Netty4Transport t = (Netty4Transport) transport;
-        @SuppressWarnings("unchecked")
         final TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) connection;
         CloseableChannel.closeChannels(channels.getChannels().subList(0, randomIntBetween(1, channels.getChannels().size())), true);
     }
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
index 5bda7e1b83d..0a09b6b8789 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
@@ -32,6 +32,7 @@ import io.netty.handler.codec.http.HttpResponseDecoder;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.HttpUtil;
 import io.netty.handler.codec.http.HttpVersion;
+
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -89,7 +90,6 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
     private final ResponseDecoder responseDecoder = new ResponseDecoder();
 
     @Before
-    @SuppressWarnings("unchecked")
     public void setMocks() {
         transport = mock(NioHttpServerTransport.class);
         Settings settings = Settings.EMPTY;
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
index 090fc579c48..9322bfd7122 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
@@ -95,7 +95,6 @@ public class SimpleNioTransportTests extends AbstractSimpleTransportTestCase {
 
     @Override
     protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
-        @SuppressWarnings("unchecked")
         TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) connection;
         CloseableChannel.closeChannels(channels.getChannels().subList(0, randomIntBetween(1, channels.getChannels().size())), true);
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsResponse.java
index 19b0517d96c..173a44b67cd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsResponse.java
@@ -48,7 +48,6 @@ public class ClusterGetSettingsResponse extends ActionResponse implements ToXCon
     static final String TRANSIENT_FIELD = "transient";
     static final String DEFAULTS_FIELD = "defaults";
 
-    @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<ClusterGetSettingsResponse, Void> PARSER =
         new ConstructingObjectParser<>(
             "cluster_get_settings_response",
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java
index e330a0b8565..d0a62fe771d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanation.java
@@ -45,7 +45,6 @@ public class QueryExplanation  implements Streamable, ToXContentFragment {
 
     public static final int RANDOM_SHARD = -1;
 
-    @SuppressWarnings("unchecked")
     static ConstructingObjectParser<QueryExplanation, Void> PARSER = new ConstructingObjectParser<>(
         "query_explanation",
         true,
diff --git a/server/src/main/java/org/elasticsearch/action/get/GetResponse.java b/server/src/main/java/org/elasticsearch/action/get/GetResponse.java
index 9ee59cf70d0..455aab7f6e3 100644
--- a/server/src/main/java/org/elasticsearch/action/get/GetResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/get/GetResponse.java
@@ -129,7 +129,6 @@ public class GetResponse extends ActionResponse implements Iterable<DocumentFiel
     /**
      * The source of the document (As a map).
      */
-    @SuppressWarnings({"unchecked"})
     public Map<String, Object> getSourceAsMap() throws ElasticsearchParseException {
         return getResult.sourceAsMap();
     }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
index 101ce7ec260..3f41aaddfb7 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
@@ -32,8 +32,8 @@ import org.elasticsearch.ingest.IngestDocument;
 
 import java.io.IOException;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class SimulateProcessorResult implements Writeable, ToXContentObject {
 
@@ -42,7 +42,6 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject {
     private final WriteableIngestDocument ingestDocument;
     private final Exception failure;
 
-    @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<ElasticsearchException, Void> IGNORED_ERROR_PARSER =
         new ConstructingObjectParser<>(
             "ignored_error_parser",
@@ -57,7 +56,6 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject {
         );
     }
 
-    @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<SimulateProcessorResult, Void> PARSER =
         new ConstructingObjectParser<>(
             "simulate_processor_result",
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
index 2430868bb59..6331097024c 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
@@ -94,7 +94,6 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment {
         );
     }
 
-    @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<WriteableIngestDocument, Void> PARSER =
         new ConstructingObjectParser<>(
             "writeable_ingest_document",
diff --git a/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java b/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java
index cdd895ff8cd..8598ab3e4be 100644
--- a/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java
+++ b/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.support;
 
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
+
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -205,7 +206,7 @@ public final class ActiveShardCount implements Writeable {
         if (o == null || getClass() != o.getClass()) {
             return false;
         }
-        @SuppressWarnings("unchecked") ActiveShardCount that = (ActiveShardCount) o;
+        ActiveShardCount that = (ActiveShardCount) o;
         return value == that.value;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
index 9dc7a899d03..7b137fb418c 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
@@ -72,7 +72,6 @@ public abstract class ReplicationRequestBuilder<Request extends ReplicationReque
      * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)}
      * to get the ActiveShardCount.
      */
-    @SuppressWarnings("unchecked")
     public RequestBuilder setWaitForActiveShards(final int waitForActiveShards) {
         return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards));
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
index 138788251c9..066f00c2cd1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
+++ b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState.Custom;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -165,7 +166,7 @@ public class RestoreInProgress extends AbstractNamedDiffable<Custom> implements
             if (o == null || getClass() != o.getClass()) {
                 return false;
             }
-            @SuppressWarnings("unchecked") Entry entry = (Entry) o;
+            Entry entry = (Entry) o;
             return snapshot.equals(entry.snapshot) &&
                        state == entry.state &&
                        indices.equals(entry.indices) &&
@@ -291,7 +292,7 @@ public class RestoreInProgress extends AbstractNamedDiffable<Custom> implements
                 return false;
             }
 
-            @SuppressWarnings("unchecked") ShardRestoreStatus status = (ShardRestoreStatus) o;
+            ShardRestoreStatus status = (ShardRestoreStatus) o;
             return state == status.state &&
                        Objects.equals(nodeId, status.nodeId) &&
                        Objects.equals(reason, status.reason);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 74789aada3a..3bb9d42a578 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -161,7 +161,6 @@ public final class IndexGraveyard implements MetaData.Custom {
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     public Diff<MetaData.Custom> diff(final MetaData.Custom previous) {
         return new IndexGraveyardDiff((IndexGraveyard) previous, this);
     }
@@ -321,7 +320,7 @@ public final class IndexGraveyard implements MetaData.Custom {
 
         @Override
         public IndexGraveyard apply(final MetaData.Custom previous) {
-            @SuppressWarnings("unchecked") final IndexGraveyard old = (IndexGraveyard) previous;
+            final IndexGraveyard old = (IndexGraveyard) previous;
             if (removedCount > old.tombstones.size()) {
                 throw new IllegalStateException("IndexGraveyardDiff cannot remove [" + removedCount + "] entries from [" +
                                                 old.tombstones.size() + "] tombstones.");
@@ -416,7 +415,7 @@ public final class IndexGraveyard implements MetaData.Custom {
             if (other == null || getClass() != other.getClass()) {
                 return false;
             }
-            @SuppressWarnings("unchecked") Tombstone that = (Tombstone) other;
+            Tombstone that = (Tombstone) other;
             return index.equals(that.index) && deleteDateInMillis == that.deleteDateInMillis;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 90380205012..18b89db72a3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -23,6 +23,7 @@ import com.carrotsearch.hppc.LongArrayList;
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -685,7 +686,6 @@ public class IndexMetaData implements Diffable<IndexMetaData>, ToXContentFragmen
                         return lookupPrototypeSafe(key).readFrom(in);
                     }
 
-                    @SuppressWarnings("unchecked")
                     @Override
                     public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
                         return lookupPrototypeSafe(key).readDiffFrom(in);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
index ae58d2885bb..d35a4baa1e6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
@@ -381,7 +381,6 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
                 aliases.build(), customs.build());
         }
 
-        @SuppressWarnings("unchecked")
         public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params)
                 throws IOException {
             builder.startObject(indexTemplateMetaData.name());
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 43339599734..c3da6388614 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster.metadata;
 import com.carrotsearch.hppc.ObjectHashSet;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.action.AliasesRequest;
@@ -169,7 +170,6 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
 
     private final SortedMap<String, AliasOrIndex> aliasAndIndexLookup;
 
-    @SuppressWarnings("unchecked")
     MetaData(String clusterUUID, long version, Settings transientSettings, Settings persistentSettings,
              ImmutableOpenMap<String, IndexMetaData> indices, ImmutableOpenMap<String, IndexTemplateMetaData> templates,
              ImmutableOpenMap<String, Custom> customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices,
@@ -1000,7 +1000,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, To
         }
 
         public IndexGraveyard indexGraveyard() {
-            @SuppressWarnings("unchecked") IndexGraveyard graveyard = (IndexGraveyard) getCustom(IndexGraveyard.TYPE);
+            IndexGraveyard graveyard = (IndexGraveyard) getCustom(IndexGraveyard.TYPE);
             return graveyard;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java
index ff7aab4a256..13cb85ea399 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java
@@ -217,7 +217,7 @@ public abstract class RecoverySource implements Writeable, ToXContentObject {
                 return false;
             }
 
-            @SuppressWarnings("unchecked") SnapshotRecoverySource that = (SnapshotRecoverySource) o;
+            SnapshotRecoverySource that = (SnapshotRecoverySource) o;
             return snapshot.equals(that.snapshot) && index.equals(that.index) && version.equals(that.version);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
index 850e8c9c142..7ce971958c9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
@@ -175,7 +175,7 @@ public abstract class AbstractAllocationDecision implements ToXContentFragment,
         if (other == null || other instanceof AbstractAllocationDecision == false) {
             return false;
         }
-        @SuppressWarnings("unchecked") AbstractAllocationDecision that = (AbstractAllocationDecision) other;
+        AbstractAllocationDecision that = (AbstractAllocationDecision) other;
         return Objects.equals(targetNode, that.targetNode) && Objects.equals(nodeDecisions, that.nodeDecisions);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
index fc2d81b38c4..c32d3e1518d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
@@ -316,7 +316,7 @@ public class AllocateUnassignedDecision extends AbstractAllocationDecision {
         if (other instanceof AllocateUnassignedDecision == false) {
             return false;
         }
-        @SuppressWarnings("unchecked") AllocateUnassignedDecision that = (AllocateUnassignedDecision) other;
+        AllocateUnassignedDecision that = (AllocateUnassignedDecision) other;
         return Objects.equals(allocationStatus, that.allocationStatus)
                    && Objects.equals(allocationId, that.allocationId)
                    && reuseStore == that.reuseStore
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
index de9795ff4c2..9439187d739 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
@@ -300,7 +300,7 @@ public final class MoveDecision extends AbstractAllocationDecision {
         if (other instanceof MoveDecision == false) {
             return false;
         }
-        @SuppressWarnings("unchecked") MoveDecision that = (MoveDecision) other;
+        MoveDecision that = (MoveDecision) other;
         return Objects.equals(allocationDecision, that.allocationDecision)
                    && Objects.equals(canRemainDecision, that.canRemainDecision)
                    && Objects.equals(clusterRebalanceDecision, that.clusterRebalanceDecision)
diff --git a/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java b/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
index de14e0cd53d..3c4b35d5c34 100644
--- a/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
+++ b/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
@@ -54,7 +54,6 @@ public abstract class AbstractLifecycleComponent extends AbstractComponent imple
         listeners.remove(listener);
     }
 
-    @SuppressWarnings({"unchecked"})
     @Override
     public void start() {
         if (!lifecycle.canMoveToStarted()) {
@@ -72,7 +71,6 @@ public abstract class AbstractLifecycleComponent extends AbstractComponent imple
 
     protected abstract void doStart();
 
-    @SuppressWarnings({"unchecked"})
     @Override
     public void stop() {
         if (!lifecycle.canMoveToStopped()) {
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ConstructorInjectorStore.java b/server/src/main/java/org/elasticsearch/common/inject/ConstructorInjectorStore.java
index ce63da62d8d..dfc216028c1 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ConstructorInjectorStore.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/ConstructorInjectorStore.java
@@ -32,7 +32,6 @@ class ConstructorInjectorStore {
     private final FailableCache<TypeLiteral<?>, ConstructorInjector<?>> cache
             = new FailableCache<TypeLiteral<?>, ConstructorInjector<?>>() {
         @Override
-        @SuppressWarnings("unchecked")
         protected ConstructorInjector<?> create(TypeLiteral<?> type, Errors errors)
                 throws ErrorsException {
             return createConstructor(type, errors);
diff --git a/server/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java b/server/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java
index e42082817c1..e2963864085 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java
@@ -101,7 +101,6 @@ class TypeConverterBindingProcessor extends AbstractProcessor {
                     },
                     new TypeConverter() {
                         @Override
-                        @SuppressWarnings("unchecked")
                         public Object convert(String value, TypeLiteral<?> toType) {
                             try {
                                 return Class.forName(value);
@@ -128,7 +127,6 @@ class TypeConverterBindingProcessor extends AbstractProcessor {
 
             TypeConverter typeConverter = new TypeConverter() {
                 @Override
-                @SuppressWarnings("unchecked")
                 public Object convert(String value, TypeLiteral<?> toType) {
                     try {
                         return parser.invoke(null, value);
diff --git a/server/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java b/server/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
index cb434a90369..d676b19dddb 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
@@ -42,7 +42,6 @@ class AssistedConstructor<T> {
     private final ParameterListKey assistedParameters;
     private final List<Parameter> allParameters;
 
-    @SuppressWarnings("unchecked")
     AssistedConstructor(Constructor<T> constructor, List<TypeLiteral<?>> parameterTypes) {
         this.constructor = constructor;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/ProviderMethod.java b/server/src/main/java/org/elasticsearch/common/inject/internal/ProviderMethod.java
index 0cfafc4a30a..349935ac7c4 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/ProviderMethod.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/internal/ProviderMethod.java
@@ -97,7 +97,7 @@ public class ProviderMethod<T> implements ProviderWithDependencies<T> {
 
         try {
             // We know this cast is safe because T is the method's return type.
-            @SuppressWarnings({"unchecked", "UnnecessaryLocalVariable"})
+            @SuppressWarnings({"unchecked"})
             T result = (T) method.invoke(instance, parameters);
             return result;
         } catch (IllegalAccessException e) {
diff --git a/server/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java b/server/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java
index 5bc1595be5f..5447f2ca399 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java
@@ -220,7 +220,6 @@ public abstract class Multibinder<T> {
         }
 
         @Override
-        @SuppressWarnings("unchecked")
         public void configure(Binder binder) {
             checkConfiguration(!isInitialized(), "Multibinder was already initialized");
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/DefaultBindingTargetVisitor.java b/server/src/main/java/org/elasticsearch/common/inject/spi/DefaultBindingTargetVisitor.java
index 75a3b615a10..0e4f7a80131 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/DefaultBindingTargetVisitor.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/spi/DefaultBindingTargetVisitor.java
@@ -78,8 +78,7 @@ public abstract class DefaultBindingTargetVisitor<T, V> implements BindingTarget
 
     // javac says it's an error to cast ProviderBinding<? extends T> to Binding<? extends T>
     @Override
-    @SuppressWarnings("unchecked")
     public V visit(ProviderBinding<? extends T> providerBinding) {
-        return visitOther((Binding<? extends T>) providerBinding);
+        return visitOther(providerBinding);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index b11aa9d4a96..d7879b0d928 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -518,7 +518,6 @@ public abstract class StreamInput extends InputStream {
         return (Map<String, Object>) readGenericValue();
     }
 
-    @SuppressWarnings({"unchecked"})
     @Nullable
     public Object readGenericValue() throws IOException {
         byte type = readByte();
diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index 021e97767d8..a3f83609037 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -178,7 +178,6 @@ public class GetResult implements Streamable, Iterable<DocumentField>, ToXConten
     /**
      * The source of the document (As a map).
      */
-    @SuppressWarnings({"unchecked"})
     public Map<String, Object> sourceAsMap() throws ElasticsearchParseException {
         if (source == null) {
             return null;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index 87942260742..a0640ac68a9 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -197,7 +197,6 @@ public class DocumentMapper implements ToXContentFragment {
         return mapping.root;
     }
 
-    @SuppressWarnings({"unchecked"})
     public <T extends MetadataFieldMapper> T metadataMapper(Class<T> type) {
         return mapping.metadataMapper(type);
     }
diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java
index 87ea08dc74d..59523f33901 100644
--- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java
+++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java
@@ -45,7 +45,6 @@ public class NodePersistentTasksExecutor {
                 task.markAsFailed(e);
             }
 
-            @SuppressWarnings("unchecked")
             @Override
             protected void doRun() throws Exception {
                 try {
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java
index 9ed0af010b5..4cb8c722f26 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java
@@ -85,7 +85,6 @@ public class PersistentTasksClusterService extends AbstractComponent implements
                 listener.onFailure(e);
             }
 
-            @SuppressWarnings("unchecked")
             @Override
             public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                 PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java
index 2ac57e074b7..a8f9c73ab32 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java
@@ -33,7 +33,6 @@ public class PersistentTasksExecutorRegistry extends AbstractComponent {
 
     private final Map<String, PersistentTasksExecutor<?>> taskExecutors;
 
-    @SuppressWarnings("unchecked")
     public PersistentTasksExecutorRegistry(Settings settings, Collection<PersistentTasksExecutor<?>> taskExecutors) {
         super(settings);
         Map<String, PersistentTasksExecutor<?>> map = new HashMap<>();
diff --git a/server/src/main/java/org/elasticsearch/repositories/IndexId.java b/server/src/main/java/org/elasticsearch/repositories/IndexId.java
index 469caa26b64..2a3d9f15d16 100644
--- a/server/src/main/java/org/elasticsearch/repositories/IndexId.java
+++ b/server/src/main/java/org/elasticsearch/repositories/IndexId.java
@@ -89,7 +89,7 @@ public final class IndexId implements Writeable, ToXContentObject {
         if (o == null || getClass() != o.getClass()) {
             return false;
         }
-        @SuppressWarnings("unchecked") IndexId that = (IndexId) o;
+        IndexId that = (IndexId) o;
         return Objects.equals(name, that.name) && Objects.equals(id, that.id);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
index 7a8d8327d5e..a97cf4bb419 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
@@ -238,7 +238,7 @@ public final class RepositoryData {
         if (obj == null || getClass() != obj.getClass()) {
             return false;
         }
-        @SuppressWarnings("unchecked") RepositoryData that = (RepositoryData) obj;
+        RepositoryData that = (RepositoryData) obj;
         return snapshotIds.equals(that.snapshotIds)
                    && snapshotStates.equals(that.snapshotStates)
                    && indices.equals(that.indices)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
index 1428a31a8de..e93266db805 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
@@ -247,7 +247,6 @@ public class InternalComposite
             this.formats = formats;
         }
 
-        @SuppressWarnings("unchecked")
         InternalBucket(StreamInput in, List<String> sourceNames, List<DocValueFormat> formats, int[] reverseMuls) throws IOException {
             this.key = new CompositeKey(in);
             this.docCount = in.readVLong();
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
index d998beedf14..c490b344bdb 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
@@ -23,7 +23,6 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -358,7 +357,6 @@ public class RangeAggregator extends BucketsAggregator {
         private final InternalRange.Factory factory;
         private final DocValueFormat format;
 
-        @SuppressWarnings("unchecked")
         public Unmapped(String name, R[] ranges, boolean keyed, DocValueFormat format,
                 SearchContext context,
                 Aggregator parent, InternalRange.Factory factory, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnPipelineAggregationBuilder.java
index d49da4658ae..185e1c63b98 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnPipelineAggregationBuilder.java
@@ -62,7 +62,6 @@ public class MovFnPipelineAggregationBuilder extends AbstractPipelineAggregation
     private static final Function<String, ConstructingObjectParser<MovFnPipelineAggregationBuilder, Void>> PARSER
         = name -> {
 
-        @SuppressWarnings("unchecked")
         ConstructingObjectParser<MovFnPipelineAggregationBuilder, Void> parser = new ConstructingObjectParser<>(
             MovFnPipelineAggregationBuilder.NAME,
             false,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
index b33ba879660..0e354e14a37 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
@@ -54,7 +54,6 @@ public class ValuesSourceConfig<VS extends ValuesSource> {
 
         if (field == null) {
             if (script == null) {
-                @SuppressWarnings("unchecked")
                 ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(ValuesSourceType.ANY);
                 config.format(resolveFormat(null, valueType));
                 return config;
diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index 30f1dfb14fc..6adad6dabf0 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -90,7 +90,6 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     private DistanceUnit unit = DistanceUnit.DEFAULT;
 
     private SortMode sortMode = null;
-    @SuppressWarnings("rawtypes")
     private QueryBuilder nestedFilter;
     private String nestedPath;
 
diff --git a/server/src/main/java/org/elasticsearch/snapshots/Snapshot.java b/server/src/main/java/org/elasticsearch/snapshots/Snapshot.java
index 314cd4053dd..2847af386b2 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/Snapshot.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/Snapshot.java
@@ -80,7 +80,7 @@ public final class Snapshot implements Writeable {
         if (o == null || getClass() != o.getClass()) {
             return false;
         }
-        @SuppressWarnings("unchecked") Snapshot that = (Snapshot) o;
+        Snapshot that = (Snapshot) o;
         return repository.equals(that.repository) && snapshotId.equals(that.snapshotId);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotId.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
index b80dfd94d75..7a8848618c2 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
@@ -22,7 +22,6 @@ package org.elasticsearch.snapshots;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -98,7 +97,7 @@ public final class SnapshotId implements Comparable<SnapshotId>, Writeable, ToXC
         if (o == null || getClass() != o.getClass()) {
             return false;
         }
-        @SuppressWarnings("unchecked") final SnapshotId that = (SnapshotId) o;
+        final SnapshotId that = (SnapshotId) o;
         return name.equals(that.name) && uuid.equals(that.uuid);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 51a4adec8d1..2d3be2435b4 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -21,7 +21,6 @@ package org.elasticsearch.threadpool;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.Counter;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -38,6 +37,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.XRejectedExecutionHandler;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.node.Node;
 
 import java.io.Closeable;
@@ -197,7 +197,7 @@ public class ThreadPool extends AbstractComponent implements Scheduler, Closeabl
         threadContext = new ThreadContext(settings);
 
         final Map<String, ExecutorHolder> executors = new HashMap<>();
-        for (@SuppressWarnings("unchecked") final Map.Entry<String, ExecutorBuilder> entry : builders.entrySet()) {
+        for (final Map.Entry<String, ExecutorBuilder> entry : builders.entrySet()) {
             final ExecutorBuilder.ExecutorSettings executorSettings = entry.getValue().getSettings(settings);
             final ExecutorHolder executorHolder = entry.getValue().build(executorSettings, threadContext);
             if (executors.containsKey(executorHolder.info.getName())) {
@@ -338,6 +338,7 @@ public class ThreadPool extends AbstractComponent implements Scheduler, Closeabl
      *         the ScheduledFuture will cannot interact with it.
      * @throws org.elasticsearch.common.util.concurrent.EsRejectedExecutionException if the task cannot be scheduled for execution
      */
+    @Override
     public ScheduledFuture<?> schedule(TimeValue delay, String executor, Runnable command) {
         if (!Names.SAME.equals(executor)) {
             command = new ThreadedRunnable(command, executor(executor));
@@ -358,6 +359,7 @@ public class ThreadPool extends AbstractComponent implements Scheduler, Closeabl
                         command, executor), e));
     }
 
+    @Override
     public Runnable preserveContext(Runnable command) {
         return getThreadContext().preserveContext(command);
     }
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportResponseHandler.java b/server/src/main/java/org/elasticsearch/transport/TransportResponseHandler.java
index 447bbd92dd2..fbe477ad04b 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportResponseHandler.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportResponseHandler.java
@@ -40,7 +40,6 @@ public interface TransportResponseHandler<T extends TransportResponse> extends W
      *
      * @return the deserialized response.
      */
-    @SuppressWarnings("deprecation")
     @Override
     default T read(StreamInput in) throws IOException {
         T instance = newInstance();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
index 264a92137be..f53eb63bc10 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
@@ -153,7 +153,6 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
         assertEquals(0, common.refresh.getListeners());
     }
 
-    @SuppressWarnings("unchecked")
     public void testUuidOnRootStatsIndices() {
         String uuid = createIndex("test").indexUUID();
         IndicesStatsResponse rsp = client().admin().indices().prepareStats().get();
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 0b9055cb758..9d82e9e1cdc 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -672,7 +672,6 @@ public class MetaDataTests extends ESTestCase {
     public static void assertLeafs(Map<String, Object> properties, String... fields) {
         for (String field : fields) {
             assertTrue(properties.containsKey(field));
-            @SuppressWarnings("unchecked")
             Map<String, Object> fieldProp = (Map<String, Object>)properties.get(field);
             assertNotNull(fieldProp);
             assertFalse(fieldProp.containsKey("properties"));
diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
index 1abdb97f174..0137f136d3e 100644
--- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -123,7 +123,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
 
         logger.info("--> make sure the node's repository can resolve the snapshots");
         final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
-        @SuppressWarnings("unchecked") final BlobStoreRepository repository =
+        final BlobStoreRepository repository =
             (BlobStoreRepository) repositoriesService.repository(repositoryName);
         final List<SnapshotId> originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
 
@@ -245,7 +245,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
         final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
-        @SuppressWarnings("unchecked") final BlobStoreRepository repository =
+        final BlobStoreRepository repository =
             (BlobStoreRepository) repositoriesService.repository(repositoryName);
         assertThat("getBlobContainer has to be lazy initialized", repository.getBlobContainer(), nullValue());
         return repository;
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
index d216709791e..2876fbbaa25 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
@@ -32,9 +32,9 @@ import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
-import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
 import org.elasticsearch.search.aggregations.metrics.max.Max;
 import org.elasticsearch.search.aggregations.metrics.stats.Stats;
@@ -84,7 +84,6 @@ public class DoubleTermsIT extends AbstractTermsTestCase {
     public static class CustomScriptPlugin extends AggregationTestScriptsPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = super.pluginScripts();
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
index bce4006fa10..e7e64027274 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
@@ -81,7 +81,6 @@ public class LongTermsIT extends AbstractTermsTestCase {
     public static class CustomScriptPlugin extends AggregationTestScriptsPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = super.pluginScripts();
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
index 4a85c2c1453..eeb6e121613 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations.bucket;
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongSet;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -77,7 +78,6 @@ public class MinDocCountIT extends AbstractTermsTestCase {
     public static class CustomScriptPlugin extends AggregationTestScriptsPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
index edc29b0d2c5..99aeac167e0 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
@@ -75,7 +75,6 @@ public class RangeIT extends ESIntegTestCase {
     public static class CustomScriptPlugin extends AggregationTestScriptsPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = super.pluginScripts();
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
index 0ed1dacb73f..b0263cb2dbd 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
@@ -1746,7 +1746,6 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
         }
     }
 
-    @SuppressWarnings("unchecked")
     private static Map<String, Object> createAfterKey(Object... fields) {
         assert fields.length % 2 == 0;
         final Map<String, Object> map = new HashMap<>();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
index 3b7e686ef4d..160e51a67b2 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
@@ -36,9 +36,6 @@ import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.AbstractTermsTestCase;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
-import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
 import org.elasticsearch.search.aggregations.metrics.stats.Stats;
@@ -103,7 +100,6 @@ public class StringTermsIT extends AbstractTermsTestCase {
     public static class CustomScriptPlugin extends AggregationTestScriptsPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = super.pluginScripts();
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
index fa46921a941..f5dc01f1914 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumAggregatorTests.java
@@ -262,7 +262,6 @@ public class CumulativeSumAggregatorTests extends AggregatorTestCase {
         });
     }
 
-    @SuppressWarnings("unchecked")
     private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consumer<InternalAggregation> verify) throws IOException {
         executeTestCase(query, aggBuilder, verify, indexWriter -> {
             Document document = new Document();
@@ -282,7 +281,6 @@ public class CumulativeSumAggregatorTests extends AggregatorTestCase {
         });
     }
 
-    @SuppressWarnings("unchecked")
     private void executeTestCase(Query query, AggregationBuilder aggBuilder, Consumer<InternalAggregation> verify,
                                  CheckedConsumer<RandomIndexWriter, IOException> setup) throws IOException {
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnUnitTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnUnitTests.java
index 88bbe3671b2..db3f2d745e1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnUnitTests.java
@@ -98,7 +98,6 @@ public class MovFnUnitTests extends AggregatorTestCase {
     }
 
 
-    @SuppressWarnings("unchecked")
     private void executeTestCase(Query query,
                                  DateHistogramAggregationBuilder aggBuilder,
                                  Consumer<Histogram> verify,
diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
index 2126e0e94eb..31fa4f838df 100644
--- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
@@ -96,7 +96,6 @@ public class SearchFieldsIT extends ESIntegTestCase {
     public static class CustomScriptPlugin extends MockScriptPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
 
@@ -143,7 +142,6 @@ public class SearchFieldsIT extends ESIntegTestCase {
             return scripts;
         }
 
-        @SuppressWarnings("unchecked")
         static Object fieldsScript(Map<String, Object> vars, String fieldName) {
             Map<?, ?> fields = (Map) vars.get("_fields");
             FieldLookup fieldLookup = (FieldLookup) fields.get(fieldName);
@@ -156,7 +154,6 @@ public class SearchFieldsIT extends ESIntegTestCase {
             return XContentMapValues.extractValue(path, source);
         }
 
-        @SuppressWarnings("unchecked")
         static Object docScript(Map<String, Object> vars, String fieldName) {
             Map<?, ?> doc = (Map) vars.get("doc");
             ScriptDocValues<?> values = (ScriptDocValues<?>) doc.get(fieldName);
diff --git a/server/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java b/server/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
index 12e48a3ae4f..fc11554dfb3 100644
--- a/server/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
+++ b/server/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.ScoreAccessor;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -48,8 +49,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction;
-
-import org.elasticsearch.script.ScriptType;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -70,7 +69,6 @@ public class FunctionScoreIT extends ESIntegTestCase {
     public static class CustomScriptPlugin extends MockScriptPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
             scripts.put("1", vars -> 1.0d);
diff --git a/server/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
index 257089c9054..8203dac1a2d 100644
--- a/server/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
+++ b/server/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java
@@ -67,7 +67,6 @@ public class RandomScoreFunctionIT extends ESIntegTestCase {
     public static class CustomScriptPlugin extends MockScriptPlugin {
 
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
 
@@ -84,7 +83,6 @@ public class RandomScoreFunctionIT extends ESIntegTestCase {
             return scripts;
         }
 
-        @SuppressWarnings("unchecked")
         static Double scoringScript(Map<String, Object> vars, Function<ScoreAccessor, Number> scoring) {
             Map<?, ?> doc = (Map) vars.get("doc");
             Double index = ((Number) ((ScriptDocValues<?>) doc.get("index")).getValues().get(0)).doubleValue();
diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index ff0196aacdf..40d6b26b4f9 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -86,7 +86,6 @@ import static org.hamcrest.Matchers.nullValue;
 public class FieldSortIT extends ESIntegTestCase {
     public static class CustomScriptPlugin extends MockScriptPlugin {
         @Override
-        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
             scripts.put("doc['number'].value", vars -> sortDoubleScript(vars));
@@ -94,14 +93,12 @@ public class FieldSortIT extends ESIntegTestCase {
             return scripts;
         }
 
-        @SuppressWarnings("unchecked")
         static Double sortDoubleScript(Map<String, Object> vars) {
             Map<?, ?> doc = (Map) vars.get("doc");
             Double index = ((Number) ((ScriptDocValues<?>) doc.get("number")).getValues().get(0)).doubleValue();
             return index;
         }
 
-        @SuppressWarnings("unchecked")
         static String sortStringScript(Map<String, Object> vars) {
             Map<?, ?> doc = (Map) vars.get("doc");
             String value = ((String) ((ScriptDocValues<?>) doc.get("keyword")).getValues().get(0));
diff --git a/server/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java b/server/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java
index aa49bed6975..6668c1be0e4 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/SimpleSortIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -50,8 +51,6 @@ import java.util.function.Function;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-
-import org.elasticsearch.script.ScriptType;
 import static org.elasticsearch.search.sort.SortBuilders.scriptSort;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -104,7 +103,6 @@ public class SimpleSortIT extends ESIntegTestCase {
         /**
          * Return the minimal value from a set of values.
          */
-        @SuppressWarnings("unchecked")
         static <T extends Comparable<T>> T getMinValueScript(Map<String, Object> vars, T initialValue, String fieldName,
                                                              Function<Object, T> converter) {
             T retval = initialValue;
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
index 439728bac9e..cef44ed17fd 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
@@ -248,7 +248,7 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
         logger.info("--> verify index folder deleted from blob container");
         RepositoriesService repositoriesSvc = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
         ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName());
-        @SuppressWarnings("unchecked") BlobStoreRepository repository = (BlobStoreRepository) repositoriesSvc.repository(repoName);
+        BlobStoreRepository repository = (BlobStoreRepository) repositoriesSvc.repository(repoName);
 
         final SetOnce<BlobContainer> indicesBlobContainer = new SetOnce<>();
         final SetOnce<RepositoryData> repositoryData = new SetOnce<>();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 5d555ece438..13540eaec95 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -29,6 +29,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
+
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -1070,7 +1071,6 @@ public abstract class ESTestCase extends LuceneTestCase {
         List<Object> targetList = new ArrayList<>();
         for(Object value : list) {
             if (value instanceof Map) {
-                @SuppressWarnings("unchecked")
                 LinkedHashMap<String, Object> valueMap = (LinkedHashMap<String, Object>) value;
                 targetList.add(shuffleMap(valueMap, exceptFields));
             } else if(value instanceof List) {
@@ -1090,7 +1090,6 @@ public abstract class ESTestCase extends LuceneTestCase {
         for (String key : keys) {
             Object value = map.get(key);
             if (value instanceof Map && exceptFields.contains(key) == false) {
-                @SuppressWarnings("unchecked")
                 LinkedHashMap<String, Object> valueMap = (LinkedHashMap<String, Object>) value;
                 targetMap.put(key, shuffleMap(valueMap, exceptFields));
             } else if(value instanceof List && exceptFields.contains(key) == false) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index cf3cc39d34d..48fc7982074 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -49,6 +49,8 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
@@ -685,7 +687,6 @@ public class ElasticsearchAssertions {
     /**
      * Compares two lists recursively, but using arrays comparisons for byte[] through Arrays.equals(byte[], byte[])
      */
-    @SuppressWarnings("unchecked")
     private static void assertListEquals(List<Object> expected, List<Object> actual) {
         assertEquals(expected.size(), actual.size());
         Iterator<Object> actualIterator = actual.iterator();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
index 39a1f1d3780..494d65e05de 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
@@ -54,7 +54,6 @@ public class GreaterThanAssertion extends Assertion {
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is greater than [{}] (field: [{}])", actualValue, expectedValue, getField());
         assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
index 84c82f4159d..6d1e5116474 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
@@ -64,7 +64,7 @@ public class MockTcpTransportTests extends AbstractSimpleTransportTestCase {
     @Override
     protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
         final MockTcpTransport t = (MockTcpTransport) transport;
-        @SuppressWarnings("unchecked") final TcpTransport.NodeChannels channels =
+        final TcpTransport.NodeChannels channels =
                 (TcpTransport.NodeChannels) connection;
         CloseableChannel.closeChannels(channels.getChannels().subList(0, randomIntBetween(1, channels.getChannels().size())), true);
     }
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java
index cf9eb5d7a8c..108411dee5b 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java
@@ -96,7 +96,6 @@ public class SimpleMockNioTransportTests extends AbstractSimpleTransportTestCase
 
     @Override
     protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
-        @SuppressWarnings("unchecked")
         TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) connection;
         CloseableChannel.closeChannels(channels.getChannels().subList(0, randomIntBetween(1, channels.getChannels().size())), true);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
index 26c35db1fc9..2668e62abbc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
@@ -100,7 +100,6 @@ public class SecurityClient {
      * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
      * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
      */
-    @SuppressWarnings("unchecked")
     public ClearRealmCacheRequestBuilder prepareClearRealmCache() {
         return new ClearRealmCacheRequestBuilder(client);
     }
@@ -109,7 +108,6 @@ public class SecurityClient {
      * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
      * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
      */
-    @SuppressWarnings("unchecked")
     public void clearRealmCache(ClearRealmCacheRequest request, ActionListener<ClearRealmCacheResponse> listener) {
         client.execute(ClearRealmCacheAction.INSTANCE, request, listener);
     }
@@ -118,7 +116,6 @@ public class SecurityClient {
      * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
      * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
      */
-    @SuppressWarnings("unchecked")
     public ActionFuture<ClearRealmCacheResponse> clearRealmCache(ClearRealmCacheRequest request) {
         return client.execute(ClearRealmCacheAction.INSTANCE, request);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeAction.java
index 84a643ae72d..89279f4ea31 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeAction.java
@@ -76,7 +76,6 @@ public class IndexUpgradeAction extends Action<BulkByScrollResponse> {
         /**
          * Sets the index.
          */
-        @SuppressWarnings("unchecked")
         public final Request index(String index) {
             this.index = index;
             return this;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
index 4be0cefe525..3e0d2d80651 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
@@ -37,7 +37,6 @@ public class MlRestTestStateCleaner {
         final Request datafeedsRequest = new Request("GET", "/_xpack/ml/datafeeds");
         datafeedsRequest.addParameter("filter_path", "datafeeds");
         final Response datafeedsResponse = adminClient.performRequest(datafeedsRequest);
-        @SuppressWarnings("unchecked")
         final List<Map<String, Object>> datafeeds =
                 (List<Map<String, Object>>) XContentMapValues.extractValue("datafeeds", ESRestTestCase.entityAsMap(datafeedsResponse));
         if (datafeeds == null) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java
index ef285b87cf1..fc2ee52dc41 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java
@@ -147,7 +147,6 @@ public class AnomalyRecordTests extends AbstractSerializingTestCase<AnomalyRecor
         assertEquals(Arrays.asList("yes", "no"), serialisedSpoilerFieldValues);
     }
 
-    @SuppressWarnings("unchecked")
     public void testToXContentDoesNotIncludesReservedWordInputFields() throws IOException {
         AnomalyRecord record = createTestInstance();
         record.setByFieldName(AnomalyRecord.BUCKET_SPAN.getPreferredName());
@@ -157,7 +156,7 @@ public class AnomalyRecordTests extends AbstractSerializingTestCase<AnomalyRecor
         XContentParser parser = createParser(XContentType.JSON.xContent(), bytes);
         Object value = parser.map().get(AnomalyRecord.BUCKET_SPAN.getPreferredName());
         assertNotEquals("bar", value);
-        assertEquals((Long)record.getBucketSpan(), (Long)value);
+        assertEquals(record.getBucketSpan(), value);
     }
 
     public void testId() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
index ae171f138cf..49515973c3b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
@@ -65,7 +65,6 @@ public class RollupRestTestStateCleaner {
     private static void deleteAllJobs(RestClient adminClient) throws Exception {
         Response response = adminClient.performRequest("GET", "/_xpack/rollup/job/_all");
         Map<String, Object> jobs = ESRestTestCase.entityAsMap(response);
-        @SuppressWarnings("unchecked")
         List<Map<String, Object>> jobConfigs =
                 (List<Map<String, Object>>) XContentMapValues.extractValue("jobs", jobs);
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java
index ef87fe392dd..e33dbc69db6 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java
@@ -43,13 +43,13 @@ import org.elasticsearch.xpack.core.ml.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
-import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.Result;
+import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery;
 import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
@@ -252,7 +252,7 @@ public class JobProviderTests extends ESTestCase {
 
         BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(1.0);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<Bucket>[] holder = new QueryPage[1];
         provider.buckets(jobId, bq, r -> holder[0] = r, e -> {throw new RuntimeException(e);}, client);
         QueryPage<Bucket> buckets = holder[0];
@@ -286,7 +286,7 @@ public class JobProviderTests extends ESTestCase {
         BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(5.1)
                 .includeInterim(true);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<Bucket>[] holder = new QueryPage[1];
         provider.buckets(jobId, bq, r -> holder[0] = r, e -> {throw new RuntimeException(e);}, client);
         QueryPage<Bucket> buckets = holder[0];
@@ -322,7 +322,7 @@ public class JobProviderTests extends ESTestCase {
         bq.anomalyScoreThreshold(5.1);
         bq.includeInterim(true);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<Bucket>[] holder = new QueryPage[1];
         provider.buckets(jobId, bq, r -> holder[0] = r, e -> {throw new RuntimeException(e);}, client);
         QueryPage<Bucket> buckets = holder[0];
@@ -368,7 +368,7 @@ public class JobProviderTests extends ESTestCase {
         BucketsQueryBuilder bq = new BucketsQueryBuilder();
         bq.timestamp(Long.toString(now.getTime()));
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<Bucket>[] bucketHolder = new QueryPage[1];
         provider.buckets(jobId, bq, q -> bucketHolder[0] = q, e -> {}, client);
         assertThat(bucketHolder[0].count(), equalTo(1L));
@@ -409,7 +409,7 @@ public class JobProviderTests extends ESTestCase {
                 .epochEnd(String.valueOf(now.getTime())).includeInterim(true).sortField(sortfield)
                 .recordScore(2.2);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<AnomalyRecord>[] holder = new QueryPage[1];
         provider.records(jobId, rqb, page -> holder[0] = page, RuntimeException::new, client);
         QueryPage<AnomalyRecord> recordPage = holder[0];
@@ -462,7 +462,7 @@ public class JobProviderTests extends ESTestCase {
         rqb.sortField(sortfield);
         rqb.recordScore(2.2);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<AnomalyRecord>[] holder = new QueryPage[1];
         provider.records(jobId, rqb, page -> holder[0] = page, RuntimeException::new, client);
         QueryPage<AnomalyRecord> recordPage = holder[0];
@@ -507,7 +507,7 @@ public class JobProviderTests extends ESTestCase {
         Client client = getMockedClient(qb -> {}, response);
         JobProvider provider = createProvider(client);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<AnomalyRecord>[] holder = new QueryPage[1];
         provider.bucketRecords(jobId, bucket, from, size, true, sortfield, true, page -> holder[0] = page, RuntimeException::new,
                 client);
@@ -568,7 +568,7 @@ public class JobProviderTests extends ESTestCase {
         Client client = getMockedClient(q -> {}, response);
 
         JobProvider provider = createProvider(client);
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<CategoryDefinition>[] holder = new QueryPage[1];
         provider.categoryDefinitions(jobId, null, false, from, size, r -> holder[0] = r,
                 e -> {throw new RuntimeException(e);}, client);
@@ -590,7 +590,7 @@ public class JobProviderTests extends ESTestCase {
         SearchResponse response = createSearchResponse(Collections.singletonList(source));
         Client client = getMockedClient(q -> {}, response);
         JobProvider provider = createProvider(client);
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<CategoryDefinition>[] holder = new QueryPage[1];
         provider.categoryDefinitions(jobId, categoryId, false, null, null,
                 r -> holder[0] = r, e -> {throw new RuntimeException(e);}, client);
@@ -632,7 +632,7 @@ public class JobProviderTests extends ESTestCase {
         Client client = getMockedClient(q -> qbHolder[0] = q, response);
         JobProvider provider = createProvider(client);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<Influencer>[] holder = new QueryPage[1];
         InfluencersQuery query = new InfluencersQueryBuilder().from(from).size(size).includeInterim(false).build();
         provider.influencers(jobId, query, page -> holder[0] = page, RuntimeException::new, client);
@@ -692,7 +692,7 @@ public class JobProviderTests extends ESTestCase {
         Client client = getMockedClient(q -> qbHolder[0] = q, response);
         JobProvider provider = createProvider(client);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<Influencer>[] holder = new QueryPage[1];
         InfluencersQuery query = new InfluencersQueryBuilder().from(from).size(size).start("0").end("0").sortField("sort")
                 .sortDescending(true).influencerScoreThreshold(0.0).includeInterim(true).build();
@@ -747,7 +747,7 @@ public class JobProviderTests extends ESTestCase {
         Client client = getMockedClient(qb -> {}, response);
         JobProvider provider = createProvider(client);
 
-        @SuppressWarnings({"unchecked", "rawtypes"})
+        @SuppressWarnings({"unchecked"})
         QueryPage<ModelSnapshot>[] holder = new QueryPage[1];
         provider.modelSnapshots(jobId, from, size, r -> holder[0] = r, RuntimeException::new);
         QueryPage<ModelSnapshot> page = holder[0];
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
index f2c18ec9d5a..c31ebd4bc2c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
@@ -195,7 +195,7 @@ public class JobResultsPersisterTests extends ESTestCase {
         verifyNoMoreInteractions(client);
     }
 
-    @SuppressWarnings({"unchecked", "rawtypes"})
+    @SuppressWarnings({"unchecked"})
     private Client mockClient(ArgumentCaptor<BulkRequest> captor) {
         Client client = mock(Client.class);
         ThreadPool threadPool = mock(ThreadPool.class);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java
index eedc42148b1..5f8b685f844 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java
@@ -65,7 +65,6 @@ public class ScoresUpdaterTests extends ESTestCase {
     }
 
     @Before
-    @SuppressWarnings("unchecked")
     public void setUpMocks() throws IOException {
         MockitoAnnotations.initMocks(this);
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
index a77f6bf24e9..efc32fccb3d 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
@@ -35,11 +35,11 @@ import org.elasticsearch.threadpool.ThreadPoolStats;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
+import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequestBuilder;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkResponse;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
 import org.elasticsearch.xpack.monitoring.LocalStateMonitoring;
 import org.elasticsearch.xpack.monitoring.MonitoringService;
 import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
@@ -112,7 +112,6 @@ public class MonitoringIT extends ESSingleNodeTestCase {
      * This test uses the Monitoring Bulk API to index document as an external application like Kibana would do. It
      * then ensure that the documents were correctly indexed and have the expected information.
      */
-    @SuppressWarnings("unchecked")
     public void testMonitoringBulk() throws Exception {
         whenExportersAreReady(() -> {
             final MonitoredSystem system = randomSystem();
@@ -188,7 +187,6 @@ public class MonitoringIT extends ESSingleNodeTestCase {
      * have been indexed with the expected information.
      */
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29880")
-    @SuppressWarnings("unchecked")
     public void testMonitoringService() throws Exception {
         final boolean createAPMIndex = randomBoolean();
         final String indexName = createAPMIndex ? "apm-2017.11.06" : "books";
@@ -284,7 +282,6 @@ public class MonitoringIT extends ESSingleNodeTestCase {
      * Asserts that the source_node information (provided as a Map) of a monitoring document correspond to
      * the current local node information
      */
-    @SuppressWarnings("unchecked")
     private void assertMonitoringDocSourceNode(final Map<String, Object> sourceNode) {
         assertEquals(6, sourceNode.size());
 
@@ -541,7 +538,6 @@ public class MonitoringIT extends ESSingleNodeTestCase {
     /**
      * Disable the monitoring service and the Local exporter.
      */
-    @SuppressWarnings("unchecked")
     public void disableMonitoring() throws Exception {
         final Settings settings = Settings.builder()
                 .putNull("xpack.monitoring.collection.enabled")
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
index 8bae951e883..3898e34b7a4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
@@ -541,7 +541,6 @@ public class AuthenticationService extends AbstractComponent {
 
         private final RestRequest request;
 
-        @SuppressWarnings("unchecked")
         AuditableRestRequest(AuditTrail auditTrail, AuthenticationFailureHandler failureHandler, ThreadContext threadContext,
                              RestRequest request) {
             super(auditTrail, failureHandler, threadContext);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
index c66ecbec2b3..67bfc2ecdcb 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
@@ -58,7 +58,6 @@ public class RestAuthenticateActionTests extends SecurityIntegTestCase {
         assertThat(response.getStatusLine().getStatusCode(), is(200));
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
         assertThat(objectPath.evaluate("username").toString(), equalTo(SecuritySettingsSource.TEST_USER_NAME));
-        @SuppressWarnings("unchecked")
         List<String> roles = objectPath.evaluate("roles");
         assertThat(roles.size(), is(1));
         assertThat(roles, contains(SecuritySettingsSource.TEST_ROLE));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SimpleSecurityNioTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SimpleSecurityNioTransportTests.java
index feca093e581..70ab085fcf7 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SimpleSecurityNioTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SimpleSecurityNioTransportTests.java
@@ -35,9 +35,6 @@ import org.elasticsearch.xpack.core.common.socket.SocketAccess;
 import org.elasticsearch.xpack.core.ssl.SSLConfiguration;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 
-import javax.net.SocketFactory;
-import javax.net.ssl.HandshakeCompletedListener;
-import javax.net.ssl.SSLSocket;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.SocketTimeoutException;
@@ -47,6 +44,10 @@ import java.util.Collections;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 
+import javax.net.SocketFactory;
+import javax.net.ssl.HandshakeCompletedListener;
+import javax.net.ssl.SSLSocket;
+
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.containsString;
@@ -119,7 +120,6 @@ public class SimpleSecurityNioTransportTests extends AbstractSimpleTransportTest
 
     @Override
     protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
-        @SuppressWarnings("unchecked")
         TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) connection;
         CloseableChannel.closeChannels(channels.getChannels().subList(0, randomIntBetween(1, channels.getChannels().size())), true);
     }
diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
index f1d9eb1fb3f..f1495f4f3ac 100644
--- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
+++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.test.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.http.HttpStatus;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.Response;
@@ -219,7 +220,6 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase {
                     @SuppressWarnings("unchecked")
                     final Map<String, Object> node = (Map<String, Object>) nodes.values().iterator().next();
 
-                    @SuppressWarnings("unchecked")
                     final Number activeWrites = (Number) extractValue("thread_pool.write.active", node);
                     return activeWrites != null && activeWrites.longValue() == 0L;
                 } catch (Exception e) {
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
index ec4e8824a19..7861557709e 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.authc.ldap;
 
 import com.unboundid.ldap.sdk.LDAPException;
 import com.unboundid.ldap.sdk.ResultCode;
+
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -302,7 +303,6 @@ public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryT
         }
     }
 
-    @SuppressWarnings("unchecked")
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/29840")
     public void testHandlingLdapReferralErrors() throws Exception {
         String groupSearchBase = "DC=ad,DC=test,DC=elasticsearch,DC=com";

From 261002283b4526a942eb8bdff0a268aaf123734b Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 23 Jul 2018 08:56:43 -0400
Subject: [PATCH 134/260] AwaitsFix RecoveryIT#testHistoryUUIDIsGenerated

Relates #31291
---
 .../src/test/java/org/elasticsearch/upgrades/RecoveryIT.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
index de5681ebe1a..062016909b6 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
@@ -47,6 +47,8 @@ import static org.hamcrest.Matchers.notNullValue;
  * In depth testing of the recovery mechanism during a rolling restart.
  */
 public class RecoveryIT extends AbstractRollingTestCase {
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31291")
     public void testHistoryUUIDIsGenerated() throws Exception {
         final String index = "index_history_uuid";
         if (CLUSTER_TYPE == ClusterType.OLD) {

From 33f11e637dc05b5112ffc8c1ca95000a4782fa6d Mon Sep 17 00:00:00 2001
From: Andrey Ershov <andrershov@gmail.com>
Date: Mon, 23 Jul 2018 16:38:55 +0200
Subject: [PATCH 135/260] Fail shard if IndexShard#storeStats runs into an
 IOException (#32241)

Fail shard if IndexShard#storeStats runs into an IOException. Closes #29008
---
 .../elasticsearch/index/shard/IndexShard.java |  1 +
 .../index/shard/IndexShardTests.java          | 85 ++++++++++++++++++-
 .../BlobStoreRepositoryRestoreTests.java      |  1 +
 .../ESIndexLevelReplicationTestCase.java      | 11 +--
 .../index/shard/IndexShardTestCase.java       | 35 +++++---
 5 files changed, 113 insertions(+), 20 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index fc08438a7d9..d4a1d0502d0 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -917,6 +917,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         try {
             return store.stats();
         } catch (IOException e) {
+            failShard("Failing shard because of exception during storeStats", e);
             throw new ElasticsearchException("io exception while building 'store stats'", e);
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 15e6151457f..1880b6b0954 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -27,6 +27,8 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.Version;
@@ -112,6 +114,7 @@ import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.FieldMaskingReader;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.ElasticsearchException;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
@@ -138,6 +141,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.LongFunction;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -1162,6 +1166,81 @@ public class IndexShardTests extends IndexShardTestCase {
         closeShards(shard);
     }
 
+
+    public void testShardStatsWithFailures() throws IOException {
+        allowShardFailures();
+        final ShardId shardId = new ShardId("index", "_na_", 0);
+        final ShardRouting shardRouting = newShardRouting(shardId, "node", true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, ShardRoutingState.INITIALIZING);
+        final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir());
+
+
+        ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .build();
+        IndexMetaData metaData = IndexMetaData.builder(shardRouting.getIndexName())
+                .settings(settings)
+                .primaryTerm(0, 1)
+                .build();
+
+        // Override two Directory methods to make them fail at our will
+        // We use AtomicReference here to inject failure in the middle of the test not immediately
+        // We use Supplier<IOException> instead of IOException to produce meaningful stacktrace
+        // (remember stack trace is filled when exception is instantiated)
+        AtomicReference<Supplier<IOException>> exceptionToThrow = new AtomicReference<>();
+        AtomicBoolean throwWhenMarkingStoreCorrupted = new AtomicBoolean(false);
+        Directory directory = new FilterDirectory(newFSDirectory(shardPath.resolveIndex())) {
+            //fileLength method is called during storeStats try block
+            //it's not called when store is marked as corrupted
+            @Override
+            public long fileLength(String name) throws IOException {
+                Supplier<IOException> ex = exceptionToThrow.get();
+                if (ex == null) {
+                    return super.fileLength(name);
+                } else {
+                    throw ex.get();
+                }
+            }
+
+            //listAll method is called when marking store as corrupted
+            @Override
+            public String[] listAll() throws IOException {
+                Supplier<IOException> ex = exceptionToThrow.get();
+                if (throwWhenMarkingStoreCorrupted.get() && ex != null) {
+                    throw ex.get();
+                } else {
+                    return super.listAll();
+                }
+            }
+        };
+
+        try (Store store = createStore(shardId, new IndexSettings(metaData, Settings.EMPTY), directory)) {
+            IndexShard shard = newShard(shardRouting, shardPath, metaData, store,
+                    null, new InternalEngineFactory(), () -> {
+                    }, EMPTY_EVENT_LISTENER);
+            AtomicBoolean failureCallbackTriggered = new AtomicBoolean(false);
+            shard.addShardFailureCallback((ig)->failureCallbackTriggered.set(true));
+
+            recoverShardFromStore(shard);
+
+            final boolean corruptIndexException = randomBoolean();
+
+            if (corruptIndexException) {
+                exceptionToThrow.set(() -> new CorruptIndexException("Test CorruptIndexException", "Test resource"));
+                throwWhenMarkingStoreCorrupted.set(randomBoolean());
+            } else {
+                exceptionToThrow.set(() -> new IOException("Test IOException"));
+            }
+            ElasticsearchException e = expectThrows(ElasticsearchException.class, shard::storeStats);
+            assertTrue(failureCallbackTriggered.get());
+
+            if (corruptIndexException && !throwWhenMarkingStoreCorrupted.get()) {
+                assertTrue(store.isMarkedCorrupted());
+            }
+        }
+    }
+
     public void testRefreshMetric() throws IOException {
         IndexShard shard = newStartedShard();
         assertThat(shard.refreshStats().getTotal(), equalTo(2L)); // refresh on: finalize and end of recovery
@@ -1868,6 +1947,7 @@ public class IndexShardTests extends IndexShardTestCase {
                 ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE),
                 shard.shardPath(),
                 shard.indexSettings().getIndexMetaData(),
+                null,
                 wrapper,
                 new InternalEngineFactory(),
                 () -> {},
@@ -2020,6 +2100,7 @@ public class IndexShardTests extends IndexShardTestCase {
                 ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE),
                 shard.shardPath(),
                 shard.indexSettings().getIndexMetaData(),
+                null,
                 wrapper,
                 new InternalEngineFactory(),
                 () -> {},
@@ -2506,7 +2587,7 @@ public class IndexShardTests extends IndexShardTestCase {
                 .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), randomFrom("false", "true", "checksum", "fix")))
             .build();
         final IndexShard newShard = newShard(shardRouting, indexShard.shardPath(), indexMetaData,
-            null, indexShard.engineFactory, indexShard.getGlobalCheckpointSyncer(), EMPTY_EVENT_LISTENER);
+                null, null, indexShard.engineFactory, indexShard.getGlobalCheckpointSyncer(), EMPTY_EVENT_LISTENER);
 
         Store.MetadataSnapshot storeFileMetaDatas = newShard.snapshotStoreMetadata();
         assertTrue("at least 2 files, commit and data: " + storeFileMetaDatas.toString(), storeFileMetaDatas.size() > 1);
@@ -3005,7 +3086,7 @@ public class IndexShardTests extends IndexShardTestCase {
         ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
         AtomicBoolean markedInactive = new AtomicBoolean();
         AtomicReference<IndexShard> primaryRef = new AtomicReference<>();
-        IndexShard primary = newShard(shardRouting, shardPath, metaData, null, new InternalEngineFactory(), () -> {
+        IndexShard primary = newShard(shardRouting, shardPath, metaData, null, null, new InternalEngineFactory(), () -> {
         }, new IndexEventListener() {
             @Override
             public void onShardInactive(IndexShard indexShard) {
diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java
index 0eae9a14200..fa7de2d6291 100644
--- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java
@@ -105,6 +105,7 @@ public class BlobStoreRepositoryRestoreTests extends IndexShardTestCase {
                     shard.shardPath(),
                     shard.indexSettings().getIndexMetaData(),
                     null,
+                    null,
                     new InternalEngineFactory(),
                     () -> {},
                     EMPTY_EVENT_LISTENER);
diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 5a5ee12065c..2b1841c39ae 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -259,13 +259,14 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
 
         public synchronized IndexShard addReplicaWithExistingPath(final ShardPath shardPath, final String nodeId) throws IOException {
             final ShardRouting shardRouting = TestShardRouting.newShardRouting(
-                shardId,
-                nodeId,
-                false, ShardRoutingState.INITIALIZING,
-                RecoverySource.PeerRecoverySource.INSTANCE);
+                    shardId,
+                    nodeId,
+                    false, ShardRoutingState.INITIALIZING,
+                    RecoverySource.PeerRecoverySource.INSTANCE);
 
             final IndexShard newReplica =
-                newShard(shardRouting, shardPath, indexMetaData, null, getEngineFactory(shardRouting), () -> {}, EMPTY_EVENT_LISTENER);
+                    newShard(shardRouting, shardPath, indexMetaData, null, null, getEngineFactory(shardRouting),
+                            () -> {}, EMPTY_EVENT_LISTENER);
             replicas.add(newReplica);
             updateAllocationIDsOnPrimary();
             return newReplica;
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index 0cbc6e44502..f9289f65861 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -163,15 +163,20 @@ public abstract class IndexShardTestCase extends ESTestCase {
         return Settings.EMPTY;
     }
 
-    private Store createStore(IndexSettings indexSettings, ShardPath shardPath) throws IOException {
-        final ShardId shardId = shardPath.getShardId();
+
+    protected Store createStore(IndexSettings indexSettings, ShardPath shardPath) throws IOException {
+        return createStore(shardPath.getShardId(), indexSettings, newFSDirectory(shardPath.resolveIndex()));
+    }
+
+    protected Store createStore(ShardId shardId, IndexSettings indexSettings, Directory directory) throws IOException {
         final DirectoryService directoryService = new DirectoryService(shardId, indexSettings) {
             @Override
             public Directory newDirectory() throws IOException {
-                return newFSDirectory(shardPath.resolveIndex());
+                return directory;
             }
         };
         return new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId));
+
     }
 
     /**
@@ -284,29 +289,32 @@ public abstract class IndexShardTestCase extends ESTestCase {
         final ShardId shardId = routing.shardId();
         final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir());
         ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
-        return newShard(routing, shardPath, indexMetaData, indexSearcherWrapper, engineFactory, globalCheckpointSyncer,
+        return newShard(routing, shardPath, indexMetaData, null, indexSearcherWrapper, engineFactory, globalCheckpointSyncer,
             EMPTY_EVENT_LISTENER, listeners);
     }
 
     /**
      * creates a new initializing shard.
-     * @param routing                shard routing to use
-     * @param shardPath              path to use for shard data
-     * @param indexMetaData          indexMetaData for the shard, including any mapping
-     * @param indexSearcherWrapper   an optional wrapper to be used during searchers
-     * @param globalCheckpointSyncer callback for syncing global checkpoints
-     * @param indexEventListener     index even listener
-     * @param listeners              an optional set of listeners to add to the shard
+     * @param routing                       shard routing to use
+     * @param shardPath                     path to use for shard data
+     * @param indexMetaData                 indexMetaData for the shard, including any mapping
+     * @param store                         an optional custom store to use. If null a default file based store will be created
+     * @param indexSearcherWrapper          an optional wrapper to be used during searchers
+     * @param globalCheckpointSyncer        callback for syncing global checkpoints
+     * @param indexEventListener            index event listener
+     * @param listeners                     an optional set of listeners to add to the shard
      */
     protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData,
-                                  @Nullable IndexSearcherWrapper indexSearcherWrapper,
+                                  @Nullable Store store, @Nullable IndexSearcherWrapper indexSearcherWrapper,
                                   @Nullable EngineFactory engineFactory,
                                   Runnable globalCheckpointSyncer,
                                   IndexEventListener indexEventListener, IndexingOperationListener... listeners) throws IOException {
         final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
         final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
         final IndexShard indexShard;
-        final Store store = createStore(indexSettings, shardPath);
+        if (store == null) {
+            store = createStore(indexSettings, shardPath);
+        }
         boolean success = false;
         try {
             IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
@@ -357,6 +365,7 @@ public abstract class IndexShardTestCase extends ESTestCase {
                 current.shardPath(),
                 current.indexSettings().getIndexMetaData(),
                 null,
+                null,
                 current.engineFactory,
                 current.getGlobalCheckpointSyncer(),
             EMPTY_EVENT_LISTENER, listeners);

From d3c4904fa38e3372e65c15d2e0278367a3c71934 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 23 Jul 2018 09:12:30 -0700
Subject: [PATCH 136/260] Painless: Clean up add methods in PainlessLookup
 (#32258)

This is largely mechanical change that cleans up the addConstructor, addMethod, and
addFields methods in PainlessLookup. Changes include renamed variables, better error
messages, and some minor code movement to make it more maintainable long term.
---
 .../lookup/PainlessLookupBuilder.java         | 609 +++++++++++-------
 .../lookup/PainlessLookupUtility.java         |  12 +-
 .../elasticsearch/painless/spi/java.lang.txt  |   2 +-
 3 files changed, 393 insertions(+), 230 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 06773d3ffdd..8945c956c27 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -24,10 +24,12 @@ import org.elasticsearch.painless.spi.WhitelistClass;
 import org.elasticsearch.painless.spi.WhitelistConstructor;
 import org.elasticsearch.painless.spi.WhitelistField;
 import org.elasticsearch.painless.spi.WhitelistMethod;
-import org.objectweb.asm.Type;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -38,8 +40,13 @@ import java.util.Objects;
 import java.util.Stack;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_TYPE_NAME;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.CONSTRUCTOR_NAME;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_CLASS_NAME;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToJavaType;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typesToCanonicalTypeNames;
 
 public class PainlessLookupBuilder {
 
@@ -123,17 +130,17 @@ public class PainlessLookupBuilder {
     private final List<Whitelist> whitelists;
 
     private final Map<String, Class<?>> canonicalClassNamesToClasses;
-    private final Map<Class<?>, PainlessClassBuilder> classesToPainlessClasses;
+    private final Map<Class<?>, PainlessClassBuilder> classesToPainlessClassBuilders;
 
     public PainlessLookupBuilder(List<Whitelist> whitelists) {
         this.whitelists = whitelists;
 
         canonicalClassNamesToClasses = new HashMap<>();
-        classesToPainlessClasses = new HashMap<>();
+        classesToPainlessClassBuilders = new HashMap<>();
 
-        canonicalClassNamesToClasses.put(DEF_TYPE_NAME, def.class);
-        classesToPainlessClasses.put(def.class,
-                new PainlessClassBuilder(DEF_TYPE_NAME, Object.class, Type.getType(Object.class)));
+        canonicalClassNamesToClasses.put(DEF_CLASS_NAME, def.class);
+        classesToPainlessClassBuilders.put(def.class,
+                new PainlessClassBuilder(DEF_CLASS_NAME, Object.class, org.objectweb.asm.Type.getType(Object.class)));
     }
 
     private Class<?> canonicalTypeNameToType(String canonicalTypeName) {
@@ -141,7 +148,7 @@ public class PainlessLookupBuilder {
     }
 
     private void validateType(Class<?> type) {
-        PainlessLookupUtility.validateType(type, classesToPainlessClasses.keySet());
+        PainlessLookupUtility.validateType(type, classesToPainlessClassBuilders.keySet());
     }
 
     public void addPainlessClass(ClassLoader classLoader, String javaClassName, boolean importClassName) {
@@ -174,10 +181,10 @@ public class PainlessLookupBuilder {
         Objects.requireNonNull(clazz);
 
         if (clazz == def.class) {
-            throw new IllegalArgumentException("cannot add reserved class [" + DEF_TYPE_NAME + "]");
+            throw new IllegalArgumentException("cannot add reserved class [" + DEF_CLASS_NAME + "]");
         }
 
-        String canonicalClassName = clazz.getCanonicalName();
+        String canonicalClassName = typeToCanonicalTypeName(clazz);
 
         if (clazz.isArray()) {
             throw new IllegalArgumentException("cannot add array type [" + canonicalClassName + "] as a class");
@@ -187,13 +194,14 @@ public class PainlessLookupBuilder {
             throw new IllegalArgumentException("invalid class name [" + canonicalClassName + "]");
         }
 
-        PainlessClassBuilder existingPainlessClassBuilder = classesToPainlessClasses.get(clazz);
+        PainlessClassBuilder existingPainlessClassBuilder = classesToPainlessClassBuilders.get(clazz);
 
         if (existingPainlessClassBuilder == null) {
-            PainlessClassBuilder painlessClassBuilder = new PainlessClassBuilder(canonicalClassName, clazz, Type.getType(clazz));
+            PainlessClassBuilder painlessClassBuilder =
+                    new PainlessClassBuilder(canonicalClassName, clazz, org.objectweb.asm.Type.getType(clazz));
 
             canonicalClassNamesToClasses.put(canonicalClassName, clazz);
-            classesToPainlessClasses.put(clazz, painlessClassBuilder);
+            classesToPainlessClassBuilders.put(clazz, painlessClassBuilder);
         } else if (existingPainlessClassBuilder.clazz.equals(clazz) == false) {
             throw new IllegalArgumentException("class [" + canonicalClassName + "] " +
                     "cannot represent multiple java classes with the same name from different class loaders");
@@ -207,308 +215,459 @@ public class PainlessLookupBuilder {
                 throw new IllegalArgumentException("must use only_fqn parameter on class [" + canonicalClassName + "] with no package");
             }
         } else {
-            Class<?> importedPainlessType = canonicalClassNamesToClasses.get(importedCanonicalClassName);
+            Class<?> importedPainlessClass = canonicalClassNamesToClasses.get(importedCanonicalClassName);
 
-            if (importedPainlessType == null) {
+            if (importedPainlessClass == null) {
                 if (importClassName) {
                     if (existingPainlessClassBuilder != null) {
-                        throw new IllegalArgumentException(
-                                "inconsistent only_fqn parameters found for painless type [" + canonicalClassName + "]");
+                        throw new IllegalArgumentException("inconsistent only_fqn parameters found for class [" + canonicalClassName + "]");
                     }
 
                     canonicalClassNamesToClasses.put(importedCanonicalClassName, clazz);
                 }
-            } else if (importedPainlessType.equals(clazz) == false) {
-                throw new IllegalArgumentException("painless type [" + importedCanonicalClassName + "] illegally represents multiple " +
-                        "java types [" + clazz.getCanonicalName() + "] and [" + importedPainlessType.getCanonicalName() + "]");
+            } else if (importedPainlessClass.equals(clazz) == false) {
+                throw new IllegalArgumentException("imported class [" + importedCanonicalClassName + "] cannot represent multiple " +
+                        "classes [" + canonicalClassName + "] and [" + typeToCanonicalTypeName(importedPainlessClass) + "]");
             } else if (importClassName == false) {
-                throw new IllegalArgumentException("inconsistent only_fqn parameters found for painless type [" + canonicalClassName + "]");
+                throw new IllegalArgumentException("inconsistent only_fqn parameters found for class [" + canonicalClassName + "]");
             }
         }
     }
 
-    private void addConstructor(String ownerStructName, WhitelistConstructor whitelistConstructor) {
-        PainlessClassBuilder ownerStruct = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(ownerStructName));
+    public void addPainlessConstructor(String targetCanonicalClassName, List<String> typeNameParameters) {
+        Objects.requireNonNull(targetCanonicalClassName);
+        Objects.requireNonNull(typeNameParameters);
 
-        if (ownerStruct == null) {
-            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for constructor with " +
-                "parameters " + whitelistConstructor.painlessParameterTypeNames);
+        Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
+
+        if (targetClass == null) {
+            throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found" +
+                    "for constructor [[" + targetCanonicalClassName + "], " + typeNameParameters  + "]");
         }
 
-        List<Class<?>> painlessParametersTypes = new ArrayList<>(whitelistConstructor.painlessParameterTypeNames.size());
-        Class<?>[] javaClassParameters = new Class<?>[whitelistConstructor.painlessParameterTypeNames.size()];
-
-        for (int parameterCount = 0; parameterCount < whitelistConstructor.painlessParameterTypeNames.size(); ++parameterCount) {
-            String painlessParameterTypeName = whitelistConstructor.painlessParameterTypeNames.get(parameterCount);
+        List<Class<?>> typeParameters = new ArrayList<>(typeNameParameters.size());
 
+        for (String typeNameParameter : typeNameParameters) {
             try {
-                Class<?> painlessParameterClass = canonicalTypeNameToType(painlessParameterTypeName);
-
-                painlessParametersTypes.add(painlessParameterClass);
-                javaClassParameters[parameterCount] = PainlessLookupUtility.typeToJavaType(painlessParameterClass);
+                Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
+                typeParameters.add(typeParameter);
             } catch (IllegalArgumentException iae) {
-                throw new IllegalArgumentException("struct not defined for constructor parameter [" + painlessParameterTypeName + "] " +
-                    "with owner struct [" + ownerStructName + "] and constructor parameters " +
-                    whitelistConstructor.painlessParameterTypeNames, iae);
+                throw new IllegalArgumentException("type parameter [" + typeNameParameter + "] not found " +
+                        "for constructor [[" + targetCanonicalClassName + "], " + typeNameParameters  + "]", iae);
             }
         }
 
-        java.lang.reflect.Constructor<?> javaConstructor;
+        addPainlessConstructor(targetClass, typeParameters);
+    }
 
-        try {
-            javaConstructor = ownerStruct.clazz.getConstructor(javaClassParameters);
-        } catch (NoSuchMethodException exception) {
-            throw new IllegalArgumentException("constructor not defined for owner struct [" + ownerStructName + "] " +
-                " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames, exception);
+    public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typeParameters) {
+        Objects.requireNonNull(targetClass);
+        Objects.requireNonNull(typeParameters);
+
+        if (targetClass == def.class) {
+            throw new IllegalArgumentException("cannot add constructor to reserved class [" + DEF_CLASS_NAME + "]");
         }
 
-        String painlessMethodKey = buildPainlessMethodKey("<init>", whitelistConstructor.painlessParameterTypeNames.size());
-        PainlessMethod painlessConstructor = ownerStruct.constructors.get(painlessMethodKey);
+        String targetCanonicalClassName = targetClass.getCanonicalName();
+        PainlessClassBuilder painlessClassBuilder = classesToPainlessClassBuilders.get(targetClass);
+
+        if (painlessClassBuilder == null) {
+            throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found" +
+                    "for constructor [[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters)  + "]");
+        }
+
+        int typeParametersSize = typeParameters.size();
+        List<Class<?>> javaTypeParameters = new ArrayList<>(typeParametersSize);
+
+        for (Class<?> typeParameter : typeParameters) {
+            try {
+                validateType(typeParameter);
+            } catch (IllegalArgumentException iae) {
+                throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] not found " +
+                        "for constructor [[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", iae);
+            }
+
+            javaTypeParameters.add(typeToJavaType(typeParameter));
+        }
+
+        Constructor<?> javaConstructor;
+
+        try {
+            javaConstructor = targetClass.getConstructor(javaTypeParameters.toArray(new Class<?>[typeParametersSize]));
+        } catch (NoSuchMethodException nsme) {
+            throw new IllegalArgumentException("constructor reflection object " +
+                    "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", nsme);
+        }
+
+        String painlessMethodKey = buildPainlessMethodKey(CONSTRUCTOR_NAME, typeParametersSize);
+        PainlessMethod painlessConstructor = painlessClassBuilder.constructors.get(painlessMethodKey);
 
         if (painlessConstructor == null) {
             org.objectweb.asm.commons.Method asmConstructor = org.objectweb.asm.commons.Method.getMethod(javaConstructor);
-            MethodHandle javaHandle;
+            MethodHandle methodHandle;
 
             try {
-                javaHandle = MethodHandles.publicLookup().in(ownerStruct.clazz).unreflectConstructor(javaConstructor);
-            } catch (IllegalAccessException exception) {
-                throw new IllegalArgumentException("constructor not defined for owner struct [" + ownerStructName + "] " +
-                    " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames);
+                methodHandle = MethodHandles.publicLookup().in(targetClass).unreflectConstructor(javaConstructor);
+            } catch (IllegalAccessException iae) {
+                throw new IllegalArgumentException("constructor method handle " +
+                        "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
             }
 
             painlessConstructor = painlessMethodCache.computeIfAbsent(
-                new PainlessMethodCacheKey(ownerStruct.clazz, "<init>", painlessParametersTypes),
-                key -> new PainlessMethod("<init>", ownerStruct.clazz, null, void.class, painlessParametersTypes,
-                    asmConstructor, javaConstructor.getModifiers(), javaHandle));
-            ownerStruct.constructors.put(painlessMethodKey, painlessConstructor);
-        } else if (painlessConstructor.arguments.equals(painlessParametersTypes) == false){
-            throw new IllegalArgumentException(
-                "illegal duplicate constructors [" + painlessMethodKey + "] found within the struct [" + ownerStruct.name + "] " +
-                    "with parameters " + painlessParametersTypes + " and " + painlessConstructor.arguments);
+                    new PainlessMethodCacheKey(targetClass, CONSTRUCTOR_NAME, typeParameters),
+                    key -> new PainlessMethod(CONSTRUCTOR_NAME, targetClass, null, void.class, typeParameters,
+                                              asmConstructor, javaConstructor.getModifiers(), methodHandle)
+            );
+
+            painlessClassBuilder.constructors.put(painlessMethodKey, painlessConstructor);
+        } else if (painlessConstructor.arguments.equals(typeParameters) == false){
+            throw new IllegalArgumentException("cannot have constructors " +
+                    "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] and " +
+                    "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(painlessConstructor.arguments) + "] " +
+                    "with the same arity and different type parameters");
         }
     }
 
-    private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, WhitelistMethod whitelistMethod) {
-        PainlessClassBuilder ownerStruct = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(ownerStructName));
+    public void addPainlessMethod(ClassLoader classLoader, String targetCanonicalClassName, String augmentedCanonicalClassName,
+            String methodName, String returnCanonicalTypeName, List<String> typeNameParameters) {
 
-        if (ownerStruct == null) {
-            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
-                "name [" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
+        Objects.requireNonNull(classLoader);
+        Objects.requireNonNull(targetCanonicalClassName);
+        Objects.requireNonNull(methodName);
+        Objects.requireNonNull(returnCanonicalTypeName);
+        Objects.requireNonNull(typeNameParameters);
+
+        Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
+
+        if (targetClass == null) {
+            throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found for method " +
+                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]");
         }
 
-        if (METHOD_NAME_PATTERN.matcher(whitelistMethod.javaMethodName).matches() == false) {
-            throw new IllegalArgumentException("invalid method name" +
-                " [" + whitelistMethod.javaMethodName + "] for owner struct [" + ownerStructName + "].");
-        }
+        Class<?> augmentedClass = null;
 
-        Class<?> javaAugmentedClass;
-
-        if (whitelistMethod.javaAugmentedClassName != null) {
+        if (augmentedCanonicalClassName != null) {
             try {
-                javaAugmentedClass = Class.forName(whitelistMethod.javaAugmentedClassName, true, whitelistClassLoader);
+                augmentedClass = Class.forName(augmentedCanonicalClassName, true, classLoader);
             } catch (ClassNotFoundException cnfe) {
-                throw new IllegalArgumentException("augmented class [" + whitelistMethod.javaAugmentedClassName + "] " +
-                    "not found for method with name [" + whitelistMethod.javaMethodName + "] " +
-                    "and parameters " + whitelistMethod.painlessParameterTypeNames, cnfe);
+                throw new IllegalArgumentException("augmented class [" + augmentedCanonicalClassName + "] not found for method " +
+                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]", cnfe);
             }
-        } else {
-            javaAugmentedClass = null;
         }
 
-        int augmentedOffset = javaAugmentedClass == null ? 0 : 1;
-
-        List<Class<?>> painlessParametersTypes = new ArrayList<>(whitelistMethod.painlessParameterTypeNames.size());
-        Class<?>[] javaClassParameters = new Class<?>[whitelistMethod.painlessParameterTypeNames.size() + augmentedOffset];
-
-        if (javaAugmentedClass != null) {
-            javaClassParameters[0] = ownerStruct.clazz;
-        }
-
-        for (int parameterCount = 0; parameterCount < whitelistMethod.painlessParameterTypeNames.size(); ++parameterCount) {
-            String painlessParameterTypeName = whitelistMethod.painlessParameterTypeNames.get(parameterCount);
+        List<Class<?>> typeParameters = new ArrayList<>(typeNameParameters.size());
 
+        for (String typeNameParameter : typeNameParameters) {
             try {
-                Class<?> painlessParameterClass = canonicalTypeNameToType(painlessParameterTypeName);
-
-                painlessParametersTypes.add(painlessParameterClass);
-                javaClassParameters[parameterCount + augmentedOffset] =
-                        PainlessLookupUtility.typeToJavaType(painlessParameterClass);
+                Class<?> typeParameter = canonicalTypeNameToType(typeNameParameter);
+                typeParameters.add(typeParameter);
             } catch (IllegalArgumentException iae) {
-                throw new IllegalArgumentException("struct not defined for method parameter [" + painlessParameterTypeName + "] " +
-                    "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
-                    "and parameters " + whitelistMethod.painlessParameterTypeNames, iae);
+                throw new IllegalArgumentException("parameter type [" + typeNameParameter + "] not found for method " +
+                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]", iae);
             }
         }
 
-        Class<?> javaImplClass = javaAugmentedClass == null ? ownerStruct.clazz : javaAugmentedClass;
-        java.lang.reflect.Method javaMethod;
+        Class<?> returnType;
 
         try {
-            javaMethod = javaImplClass.getMethod(whitelistMethod.javaMethodName, javaClassParameters);
-        } catch (NoSuchMethodException nsme) {
-            throw new IllegalArgumentException("method with name [" + whitelistMethod.javaMethodName + "] " +
-                "and parameters " + whitelistMethod.painlessParameterTypeNames + " not found for class [" +
-                javaImplClass.getName() + "]", nsme);
-        }
-
-        Class<?> painlessReturnClass;
-
-        try {
-            painlessReturnClass = canonicalTypeNameToType(whitelistMethod.painlessReturnTypeName);
+            returnType = canonicalTypeNameToType(returnCanonicalTypeName);
         } catch (IllegalArgumentException iae) {
-            throw new IllegalArgumentException("struct not defined for return type [" + whitelistMethod.painlessReturnTypeName + "] " +
-                "with owner struct [" + ownerStructName + "] and method with name [" + whitelistMethod.javaMethodName + "] " +
-                "and parameters " + whitelistMethod.painlessParameterTypeNames, iae);
+            throw new IllegalArgumentException("parameter type [" + returnCanonicalTypeName + "] not found for method " +
+                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typeNameParameters + "]", iae);
         }
 
-        if (javaMethod.getReturnType() != PainlessLookupUtility.typeToJavaType(painlessReturnClass)) {
-            throw new IllegalArgumentException("specified return type class [" + painlessReturnClass + "] " +
-                "does not match the return type class [" + javaMethod.getReturnType() + "] for the " +
-                "method with name [" + whitelistMethod.javaMethodName + "] " +
-                "and parameters " + whitelistMethod.painlessParameterTypeNames);
+        addPainlessMethod(targetClass, augmentedClass, methodName, returnType, typeParameters);
+    }
+
+    public void addPainlessMethod(Class<?> targetClass, Class<?> augmentedClass, String methodName,
+            Class<?> returnType, List<Class<?>> typeParameters) {
+        Objects.requireNonNull(targetClass);
+        Objects.requireNonNull(methodName);
+        Objects.requireNonNull(returnType);
+        Objects.requireNonNull(typeParameters);
+
+        if (targetClass == def.class) {
+            throw new IllegalArgumentException("cannot add method to reserved class [" + DEF_CLASS_NAME + "]");
         }
 
-        String painlessMethodKey =
-            buildPainlessMethodKey(whitelistMethod.javaMethodName, whitelistMethod.painlessParameterTypeNames.size());
+        String targetCanonicalClassName = typeToCanonicalTypeName(targetClass);
 
-        if (javaAugmentedClass == null && Modifier.isStatic(javaMethod.getModifiers())) {
-            PainlessMethod painlessMethod = ownerStruct.staticMethods.get(painlessMethodKey);
+        if (METHOD_NAME_PATTERN.matcher(methodName).matches() == false) {
+            throw new IllegalArgumentException(
+                    "invalid method name [" + methodName + "] for target class [" + targetCanonicalClassName + "].");
+        }
 
-            if (painlessMethod == null) {
-                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
-                MethodHandle javaMethodHandle;
+        PainlessClassBuilder painlessClassBuilder = classesToPainlessClassBuilders.get(targetClass);
 
-                try {
-                    javaMethodHandle = MethodHandles.publicLookup().in(javaImplClass).unreflect(javaMethod);
-                } catch (IllegalAccessException exception) {
-                    throw new IllegalArgumentException("method handle not found for method with name " +
-                        "[" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
-                }
+        if (painlessClassBuilder == null) {
+            throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found for method " +
+                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]");
+        }
 
-                painlessMethod = painlessMethodCache.computeIfAbsent(
-                    new PainlessMethodCacheKey(ownerStruct.clazz, whitelistMethod.javaMethodName, painlessParametersTypes),
-                    key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct.clazz, null, painlessReturnClass,
-                        painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
-                ownerStruct.staticMethods.put(painlessMethodKey, painlessMethod);
-            } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn == painlessReturnClass &&
-                painlessMethod.arguments.equals(painlessParametersTypes)) == false) {
-                throw new IllegalArgumentException("illegal duplicate static methods [" + painlessMethodKey + "] " +
-                    "found within the struct [" + ownerStruct.name + "] with name [" + whitelistMethod.javaMethodName + "], " +
-                    "return types [" + painlessReturnClass + "] and [" + painlessMethod.rtn + "], " +
-                    "and parameters " + painlessParametersTypes + " and " + painlessMethod.arguments);
+        int typeParametersSize = typeParameters.size();
+        int augmentedParameterOffset = augmentedClass == null ? 0 : 1;
+        List<Class<?>> javaTypeParameters = new ArrayList<>(typeParametersSize + augmentedParameterOffset);
+
+        if (augmentedClass != null) {
+            javaTypeParameters.add(targetClass);
+        }
+
+        for (Class<?> typeParameter : typeParameters) {
+            try {
+                validateType(typeParameter);
+            } catch (IllegalArgumentException iae) {
+                throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] " +
+                        "not found for method [[" + targetCanonicalClassName + "], [" + methodName + "], " +
+                        typesToCanonicalTypeNames(typeParameters) + "]", iae);
+            }
+
+            javaTypeParameters.add(typeToJavaType(typeParameter));
+        }
+
+        try {
+            validateType(returnType);
+        } catch (IllegalArgumentException iae) {
+            throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(returnType) + "] not found for method " +
+                    "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", iae);
+        }
+
+        Method javaMethod;
+
+        if (augmentedClass == null) {
+            try {
+                javaMethod = targetClass.getMethod(methodName, javaTypeParameters.toArray(new Class<?>[typeParametersSize]));
+            } catch (NoSuchMethodException nsme) {
+                throw new IllegalArgumentException("method reflection object [[" + targetCanonicalClassName + "], " +
+                        "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", nsme);
             }
         } else {
-            PainlessMethod painlessMethod = ownerStruct.methods.get(painlessMethodKey);
+            try {
+                javaMethod = augmentedClass.getMethod(methodName, javaTypeParameters.toArray(new Class<?>[typeParametersSize]));
+            } catch (NoSuchMethodException nsme) {
+                throw new IllegalArgumentException("method reflection object [[" + targetCanonicalClassName + "], " +
+                        "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found " +
+                        "with augmented target class [" + typeToCanonicalTypeName(augmentedClass) + "]", nsme);
+            }
+        }
+
+        if (javaMethod.getReturnType() != typeToJavaType(returnType)) {
+            throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(javaMethod.getReturnType()) + "] " +
+                    "does not match the specified returned type [" + typeToCanonicalTypeName(returnType) + "] " +
+                    "for method [[" + targetClass.getCanonicalName() + "], [" + methodName + "], " +
+                    typesToCanonicalTypeNames(typeParameters) + "]");
+        }
+
+        String painlessMethodKey = buildPainlessMethodKey(methodName, typeParametersSize);
+
+        if (augmentedClass == null && Modifier.isStatic(javaMethod.getModifiers())) {
+            PainlessMethod painlessMethod = painlessClassBuilder.staticMethods.get(painlessMethodKey);
 
             if (painlessMethod == null) {
                 org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
                 MethodHandle javaMethodHandle;
 
                 try {
-                    javaMethodHandle = MethodHandles.publicLookup().in(javaImplClass).unreflect(javaMethod);
-                } catch (IllegalAccessException exception) {
-                    throw new IllegalArgumentException("method handle not found for method with name " +
-                        "[" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
+                    javaMethodHandle = MethodHandles.publicLookup().in(targetClass).unreflect(javaMethod);
+                } catch (IllegalAccessException iae) {
+                    throw new IllegalArgumentException("static method handle [[" + targetClass.getCanonicalName() + "], " +
+                            "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
                 }
 
                 painlessMethod = painlessMethodCache.computeIfAbsent(
-                    new PainlessMethodCacheKey(ownerStruct.clazz, whitelistMethod.javaMethodName, painlessParametersTypes),
-                    key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct.clazz, javaAugmentedClass, painlessReturnClass,
-                        painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
-                ownerStruct.methods.put(painlessMethodKey, painlessMethod);
-            } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn.equals(painlessReturnClass) &&
-                painlessMethod.arguments.equals(painlessParametersTypes)) == false) {
-                throw new IllegalArgumentException("illegal duplicate member methods [" + painlessMethodKey + "] " +
-                    "found within the struct [" + ownerStruct.name + "] with name [" + whitelistMethod.javaMethodName + "], " +
-                    "return types [" + painlessReturnClass + "] and [" + painlessMethod.rtn + "], " +
-                    "and parameters " + painlessParametersTypes + " and " + painlessMethod.arguments);
+                        new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
+                        key -> new PainlessMethod(methodName, targetClass, null, returnType,
+                                typeParameters, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
+
+                painlessClassBuilder.staticMethods.put(painlessMethodKey, painlessMethod);
+            } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
+                    painlessMethod.arguments.equals(typeParameters)) == false) {
+                throw new IllegalArgumentException("cannot have static methods " +
+                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
+                        "[" + typeToCanonicalTypeName(returnType) + "], " +
+                        typesToCanonicalTypeNames(typeParameters) + "] and " +
+                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
+                        "[" + typeToCanonicalTypeName(painlessMethod.rtn) + "], " +
+                        typesToCanonicalTypeNames(painlessMethod.arguments) + "] " +
+                        "with the same arity and different return type or type parameters");
+            }
+        } else {
+            PainlessMethod painlessMethod = painlessClassBuilder.staticMethods.get(painlessMethodKey);
+
+            if (painlessMethod == null) {
+                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
+                MethodHandle javaMethodHandle;
+
+                if (augmentedClass == null) {
+                    try {
+                        javaMethodHandle = MethodHandles.publicLookup().in(targetClass).unreflect(javaMethod);
+                    } catch (IllegalAccessException iae) {
+                        throw new IllegalArgumentException("method handle [[" + targetClass.getCanonicalName() + "], " +
+                                "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
+                    }
+                } else {
+                    try {
+                        javaMethodHandle = MethodHandles.publicLookup().in(augmentedClass).unreflect(javaMethod);
+                    } catch (IllegalAccessException iae) {
+                        throw new IllegalArgumentException("method handle [[" + targetClass.getCanonicalName() + "], " +
+                                "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found " +
+                                "with augmented target class [" + typeToCanonicalTypeName(augmentedClass) + "]", iae);
+                    }
+                }
+
+                painlessMethod = painlessMethodCache.computeIfAbsent(
+                        new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
+                        key -> new PainlessMethod(methodName, targetClass, augmentedClass, returnType,
+                                typeParameters, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
+
+                painlessClassBuilder.methods.put(painlessMethodKey, painlessMethod);
+            } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
+                    painlessMethod.arguments.equals(typeParameters)) == false) {
+                throw new IllegalArgumentException("cannot have methods " +
+                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
+                        "[" + typeToCanonicalTypeName(returnType) + "], " +
+                        typesToCanonicalTypeNames(typeParameters) + "] and " +
+                        "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
+                        "[" + typeToCanonicalTypeName(painlessMethod.rtn) + "], " +
+                        typesToCanonicalTypeNames(painlessMethod.arguments) + "] " +
+                        "with the same arity and different return type or type parameters");
             }
         }
     }
 
-    private void addField(String ownerStructName, WhitelistField whitelistField) {
-        PainlessClassBuilder ownerStruct = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(ownerStructName));
+    public void addPainlessField(String targetCanonicalClassName, String fieldName, String typeNameParameter) {
+        Objects.requireNonNull(targetCanonicalClassName);
+        Objects.requireNonNull(fieldName);
+        Objects.requireNonNull(typeNameParameter);
 
-        if (ownerStruct == null) {
-            throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " +
-                "name [" + whitelistField.javaFieldName + "] and type " + whitelistField.painlessFieldTypeName);
+        Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
+
+        if (targetClass == null) {
+            throw new IllegalArgumentException("class [" + targetCanonicalClassName + "] not found");
         }
 
-        if (FIELD_NAME_PATTERN.matcher(whitelistField.javaFieldName).matches() == false) {
-            throw new IllegalArgumentException("invalid field name " +
-                "[" + whitelistField.painlessFieldTypeName + "] for owner struct [" + ownerStructName + "].");
-        }
-
-        java.lang.reflect.Field javaField;
+        Class<?> typeParameter;
 
         try {
-            javaField = ownerStruct.clazz.getField(whitelistField.javaFieldName);
-        } catch (NoSuchFieldException exception) {
-            throw new IllegalArgumentException("field [" + whitelistField.javaFieldName + "] " +
-                "not found for class [" + ownerStruct.clazz.getName() + "].");
-        }
-
-        Class<?> painlessFieldClass;
-
-        try {
-            painlessFieldClass = canonicalTypeNameToType(whitelistField.painlessFieldTypeName);
+            typeParameter = canonicalTypeNameToType(typeNameParameter);
         } catch (IllegalArgumentException iae) {
-            throw new IllegalArgumentException("struct not defined for return type [" + whitelistField.painlessFieldTypeName + "] " +
-                "with owner struct [" + ownerStructName + "] and field with name [" + whitelistField.javaFieldName + "]", iae);
+            throw new IllegalArgumentException("type parameter [" + typeNameParameter + "] not found " +
+                    "for field [[" + targetCanonicalClassName + "], [" + fieldName + "]");
         }
 
+
+        addPainlessField(targetClass, fieldName, typeParameter);
+    }
+
+    public void addPainlessField(Class<?> targetClass, String fieldName, Class<?> typeParameter) {
+        Objects.requireNonNull(targetClass);
+        Objects.requireNonNull(fieldName);
+        Objects.requireNonNull(typeParameter);
+
+        if (targetClass == def.class) {
+            throw new IllegalArgumentException("cannot add field to reserved class [" + DEF_CLASS_NAME + "]");
+        }
+
+        String targetCanonicalClassName = typeToCanonicalTypeName(targetClass);
+
+        if (FIELD_NAME_PATTERN.matcher(fieldName).matches() == false) {
+            throw new IllegalArgumentException(
+                    "invalid field name [" + fieldName + "] for target class [" + targetCanonicalClassName + "].");
+        }
+
+
+        PainlessClassBuilder painlessClassBuilder = classesToPainlessClassBuilders.get(targetClass);
+
+        if (painlessClassBuilder == null) {
+            throw new IllegalArgumentException("class [" + targetCanonicalClassName + "] not found");
+        }
+
+        try {
+            validateType(typeParameter);
+        } catch (IllegalArgumentException iae) {
+            throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] not found " +
+                    "for field [[" + targetCanonicalClassName + "], [" + fieldName + "]", iae);
+        }
+
+        Field javaField;
+
+        try {
+            javaField = targetClass.getField(fieldName);
+        } catch (NoSuchFieldException nsme) {
+            throw new IllegalArgumentException(
+                    "field reflection object [[" + targetCanonicalClassName + "], [" + fieldName + "] not found", nsme);
+        }
+
+        if (javaField.getType() != typeToJavaType(typeParameter)) {
+            throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(javaField.getType()) + "] " +
+                    "does not match the specified type parameter [" + typeToCanonicalTypeName(typeParameter) + "] " +
+                    "for field [[" + targetCanonicalClassName + "], [" + fieldName + "]");
+        }
+
+        String painlessFieldKey = buildPainlessFieldKey(fieldName);
+
         if (Modifier.isStatic(javaField.getModifiers())) {
             if (Modifier.isFinal(javaField.getModifiers()) == false) {
-                throw new IllegalArgumentException("static [" + whitelistField.javaFieldName + "] " +
-                    "with owner struct [" + ownerStruct.name + "] is not final");
+                throw new IllegalArgumentException("static field [[" + targetCanonicalClassName + "]. [" + fieldName + "]] must be final");
             }
 
-            PainlessField painlessField = ownerStruct.staticMembers.get(whitelistField.javaFieldName);
+            PainlessField painlessField = painlessClassBuilder.staticMembers.get(painlessFieldKey);
 
             if (painlessField == null) {
                 painlessField = painlessFieldCache.computeIfAbsent(
-                    new PainlessFieldCacheKey(ownerStruct.clazz, whitelistField.javaFieldName, painlessFieldClass),
-                    key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
-                        ownerStruct.clazz, painlessFieldClass, javaField.getModifiers(), null, null));
-                ownerStruct.staticMembers.put(whitelistField.javaFieldName, painlessField);
-            } else if (painlessField.clazz != painlessFieldClass) {
-                throw new IllegalArgumentException("illegal duplicate static fields [" + whitelistField.javaFieldName + "] " +
-                    "found within the struct [" + ownerStruct.name + "] with type [" + whitelistField.painlessFieldTypeName + "]");
+                        new PainlessFieldCacheKey(targetClass, fieldName, typeParameter),
+                        key -> new PainlessField(fieldName, javaField.getName(), targetClass,
+                                typeParameter, javaField.getModifiers(), null, null));
+
+                painlessClassBuilder.staticMembers.put(painlessFieldKey, painlessField);
+            } else if (painlessField.clazz != typeParameter) {
+                throw new IllegalArgumentException("cannot have static fields " +
+                        "[[" + targetCanonicalClassName + "], [" + fieldName + "], [" +
+                        typeToCanonicalTypeName(typeParameter) + "] and " +
+                        "[[" + targetCanonicalClassName + "], [" + painlessField.name + "], " +
+                        typeToCanonicalTypeName(painlessField.clazz) + "] " +
+                        "with the same and different type parameters");
             }
         } else {
-            MethodHandle javaMethodHandleGetter;
-            MethodHandle javaMethodHandleSetter;
+            MethodHandle methodHandleGetter;
 
             try {
-                if (Modifier.isStatic(javaField.getModifiers()) == false) {
-                    javaMethodHandleGetter = MethodHandles.publicLookup().unreflectGetter(javaField);
-                    javaMethodHandleSetter = MethodHandles.publicLookup().unreflectSetter(javaField);
-                } else {
-                    javaMethodHandleGetter = null;
-                    javaMethodHandleSetter = null;
-                }
-            } catch (IllegalAccessException exception) {
-                throw new IllegalArgumentException("getter/setter [" + whitelistField.javaFieldName + "]" +
-                    " not found for class [" + ownerStruct.clazz.getName() + "].");
+                methodHandleGetter = MethodHandles.publicLookup().unreflectGetter(javaField);
+            } catch (IllegalAccessException iae) {
+                throw new IllegalArgumentException(
+                        "method handle getter not found for field [[" + targetCanonicalClassName + "], [" + fieldName + "]]");
             }
 
-            PainlessField painlessField = ownerStruct.members.get(whitelistField.javaFieldName);
+            MethodHandle methodHandleSetter;
+
+            try {
+                methodHandleSetter = MethodHandles.publicLookup().unreflectSetter(javaField);
+            } catch (IllegalAccessException iae) {
+                throw new IllegalArgumentException(
+                        "method handle setter not found for field [[" + targetCanonicalClassName + "], [" + fieldName + "]]");
+            }
+
+            PainlessField painlessField = painlessClassBuilder.members.get(painlessFieldKey);
 
             if (painlessField == null) {
                 painlessField = painlessFieldCache.computeIfAbsent(
-                    new PainlessFieldCacheKey(ownerStruct.clazz, whitelistField.javaFieldName, painlessFieldClass),
-                    key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(),
-                        ownerStruct.clazz, painlessFieldClass, javaField.getModifiers(), javaMethodHandleGetter, javaMethodHandleSetter));
-                ownerStruct.members.put(whitelistField.javaFieldName, painlessField);
-            } else if (painlessField.clazz != painlessFieldClass) {
-                throw new IllegalArgumentException("illegal duplicate member fields [" + whitelistField.javaFieldName + "] " +
-                    "found within the struct [" + ownerStruct.name + "] with type [" + whitelistField.painlessFieldTypeName + "]");
+                        new PainlessFieldCacheKey(targetClass, painlessFieldKey, typeParameter),
+                        key -> new PainlessField(fieldName, javaField.getName(), targetClass,
+                                typeParameter, javaField.getModifiers(), methodHandleGetter, methodHandleSetter));
+
+                painlessClassBuilder.members.put(fieldName, painlessField);
+            } else if (painlessField.clazz != typeParameter) {
+                throw new IllegalArgumentException("cannot have fields " +
+                        "[[" + targetCanonicalClassName + "], [" + fieldName + "], [" +
+                        typeToCanonicalTypeName(typeParameter) + "] and " +
+                        "[[" + targetCanonicalClassName + "], [" + painlessField.name + "], " +
+                        typeToCanonicalTypeName(painlessField.clazz) + "] " +
+                        "with the same and different type parameters");
             }
         }
     }
 
     private void copyStruct(String struct, List<String> children) {
-        final PainlessClassBuilder owner = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(struct));
+        final PainlessClassBuilder owner = classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(struct));
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
@@ -516,7 +675,7 @@ public class PainlessLookupBuilder {
 
         for (int count = 0; count < children.size(); ++count) {
             final PainlessClassBuilder child =
-                    classesToPainlessClasses.get(canonicalClassNamesToClasses.get(children.get(count)));
+                    classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(children.get(count)));
 
             if (child == null) {
                 throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
@@ -690,7 +849,7 @@ public class PainlessLookupBuilder {
                 for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
                     String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
                     PainlessClassBuilder painlessStruct =
-                        classesToPainlessClasses.get(canonicalClassNamesToClasses.get(painlessTypeName));
+                        classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(painlessTypeName));
 
                     if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
                         throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
@@ -701,8 +860,8 @@ public class PainlessLookupBuilder {
                     addPainlessClass(
                             whitelist.javaClassLoader, whitelistStruct.javaClassName, whitelistStruct.onlyFQNJavaClassName == false);
 
-                    painlessStruct = classesToPainlessClasses.get(canonicalClassNamesToClasses.get(painlessTypeName));
-                    classesToPainlessClasses.put(painlessStruct.clazz, painlessStruct);
+                    painlessStruct = classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(painlessTypeName));
+                    classesToPainlessClassBuilders.put(painlessStruct.clazz, painlessStruct);
                 }
             }
 
@@ -715,17 +874,19 @@ public class PainlessLookupBuilder {
 
                     for (WhitelistConstructor whitelistConstructor : whitelistStruct.whitelistConstructors) {
                         origin = whitelistConstructor.origin;
-                        addConstructor(painlessTypeName, whitelistConstructor);
+                        addPainlessConstructor(painlessTypeName, whitelistConstructor.painlessParameterTypeNames);
                     }
 
                     for (WhitelistMethod whitelistMethod : whitelistStruct.whitelistMethods) {
                         origin = whitelistMethod.origin;
-                        addMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod);
+                        addPainlessMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod.javaAugmentedClassName,
+                                whitelistMethod.javaMethodName, whitelistMethod.painlessReturnTypeName,
+                                whitelistMethod.painlessParameterTypeNames);
                     }
 
                     for (WhitelistField whitelistField : whitelistStruct.whitelistFields) {
                         origin = whitelistField.origin;
-                        addField(painlessTypeName, whitelistField);
+                        addPainlessField(painlessTypeName, whitelistField.javaFieldName, whitelistField.painlessFieldTypeName);
                     }
                 }
             }
@@ -735,8 +896,8 @@ public class PainlessLookupBuilder {
 
         // goes through each Painless struct and determines the inheritance list,
         // and then adds all inherited types to the Painless struct's whitelist
-        for (Class<?> javaClass : classesToPainlessClasses.keySet()) {
-            PainlessClassBuilder painlessStruct = classesToPainlessClasses.get(javaClass);
+        for (Class<?> javaClass : classesToPainlessClassBuilders.keySet()) {
+            PainlessClassBuilder painlessStruct = classesToPainlessClassBuilders.get(javaClass);
 
             List<String> painlessSuperStructs = new ArrayList<>();
             Class<?> javaSuperClass = painlessStruct.clazz.getSuperclass();
@@ -747,7 +908,7 @@ public class PainlessLookupBuilder {
             // adds super classes to the inheritance list
             if (javaSuperClass != null && javaSuperClass.isInterface() == false) {
                 while (javaSuperClass != null) {
-                    PainlessClassBuilder painlessSuperStruct = classesToPainlessClasses.get(javaSuperClass);
+                    PainlessClassBuilder painlessSuperStruct = classesToPainlessClassBuilders.get(javaSuperClass);
 
                     if (painlessSuperStruct != null) {
                         painlessSuperStructs.add(painlessSuperStruct.name);
@@ -763,7 +924,7 @@ public class PainlessLookupBuilder {
                 Class<?> javaInterfaceLookup = javaInteraceLookups.pop();
 
                 for (Class<?> javaSuperInterface : javaInterfaceLookup.getInterfaces()) {
-                    PainlessClassBuilder painlessInterfaceStruct = classesToPainlessClasses.get(javaSuperInterface);
+                    PainlessClassBuilder painlessInterfaceStruct = classesToPainlessClassBuilders.get(javaSuperInterface);
 
                     if (painlessInterfaceStruct != null) {
                         String painlessInterfaceStructName = painlessInterfaceStruct.name;
@@ -784,7 +945,7 @@ public class PainlessLookupBuilder {
 
             // copies methods and fields from Object into interface types
             if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) {
-                PainlessClassBuilder painlessObjectStruct = classesToPainlessClasses.get(Object.class);
+                PainlessClassBuilder painlessObjectStruct = classesToPainlessClassBuilders.get(Object.class);
 
                 if (painlessObjectStruct != null) {
                     copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name));
@@ -793,14 +954,14 @@ public class PainlessLookupBuilder {
         }
 
         // precompute runtime classes
-        for (PainlessClassBuilder painlessStruct : classesToPainlessClasses.values()) {
+        for (PainlessClassBuilder painlessStruct : classesToPainlessClassBuilders.values()) {
             addRuntimeClass(painlessStruct);
         }
 
         Map<Class<?>, PainlessClass> javaClassesToPainlessClasses = new HashMap<>();
 
         // copy all structs to make them unmodifiable for outside users:
-        for (Map.Entry<Class<?>,PainlessClassBuilder> entry : classesToPainlessClasses.entrySet()) {
+        for (Map.Entry<Class<?>,PainlessClassBuilder> entry : classesToPainlessClassBuilders.entrySet()) {
             entry.getValue().functionalMethod = computeFunctionalInterfaceMethod(entry.getValue());
             javaClassesToPainlessClasses.put(entry.getKey(), entry.getValue().build());
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
index 1f698b7c673..86d3f876638 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
@@ -33,10 +33,12 @@ import java.util.Objects;
  *
  * A class is a set of methods and fields under a specific class name. A type is either a class or an array under a specific type name.
  * Note the distinction between class versus type is class means that no array classes will be be represented whereas type allows array
- * classes to be represented.  The set of available classes will always be a subset of the available types.
+ * classes to be represented. The set of available classes will always be a subset of the available types.
  *
  * Under ambiguous circumstances most variable names are prefixed with asm, java, or painless. If the variable value is the same for asm,
- * java, and painless, no prefix is used.
+ * java, and painless, no prefix is used. Target is used as a prefix to represent if a constructor, method, or field is being
+ * called/accessed on that specific class.  Parameter is often a postfix used to represent if a type is used as a parameter to a
+ * constructor, method, or field.
  *
  * <ul>
  *     <li> - javaClassName     (String)         - the fully qualified java class name where '$' tokens represent inner classes excluding
@@ -150,8 +152,8 @@ public final class PainlessLookupUtility {
 
         String canonicalTypeName = type.getCanonicalName();
 
-        if (canonicalTypeName.startsWith(def.class.getName())) {
-            canonicalTypeName = canonicalTypeName.replace(def.class.getName(), DEF_TYPE_NAME);
+        if (canonicalTypeName.startsWith(def.class.getCanonicalName())) {
+            canonicalTypeName = canonicalTypeName.replace(def.class.getCanonicalName(), DEF_CLASS_NAME);
         }
 
         return canonicalTypeName;
@@ -351,7 +353,7 @@ public final class PainlessLookupUtility {
     /**
      * The def type name as specified in the source for a script.
      */
-    public static final String DEF_TYPE_NAME = "def";
+    public static final String DEF_CLASS_NAME = "def";
 
     /**
      * The method name for all constructors.
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.lang.txt
index a793ef847f9..ef2d462127f 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.lang.txt
@@ -148,7 +148,7 @@ class java.lang.Character {
   int MAX_RADIX
   char MAX_SURROGATE
   char MAX_VALUE
-  char MIN_CODE_POINT
+  int MIN_CODE_POINT
   char MIN_HIGH_SURROGATE
   char MIN_LOW_SURROGATE
   int MIN_RADIX

From d43c3a18b9e729c4d3a9ad3be8efe32eb1722c09 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 23 Jul 2018 10:57:40 -0700
Subject: [PATCH 137/260] awaitsfix SSLConfigurationReloaderTests

---
 .../xpack/core/ssl/SSLConfigurationReloaderTests.java           | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
index d8e0b693f70..3e36550e46f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
@@ -78,6 +78,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
     /**
      * Tests reloading a keystore that is used in the KeyManager of SSLContext
      */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32124")
     public void testReloadingKeyStore() throws Exception {
         assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         final Path tempDir = createTempDir();
@@ -191,6 +192,7 @@ public class SSLConfigurationReloaderTests extends ESTestCase {
      * Tests the reloading of SSLContext when the trust store is modified. The same store is used as a TrustStore (for the
      * reloadable SSLContext used in the HTTPClient) and as a KeyStore for the MockWebServer
      */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32124")
     public void testReloadingTrustStore() throws Exception {
         assumeFalse("Can't run in a FIPS JVM", inFipsJvm());
         Path tempDir = createTempDir();

From 4804da06bf6475a2b2d37587e60df46b8bf00b69 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 Jul 2018 14:07:18 -0400
Subject: [PATCH 138/260] Switch x-pack:core to new style Requests (#32252)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `x-pack:core` project to use the new versions.
---
 .../license/StartBasicLicenseTests.java       | 16 +++++----
 .../license/StartTrialLicenseTests.java       | 23 ++++++++-----
 .../integration/MlRestTestStateCleaner.java   | 33 ++++---------------
 .../rollup/RollupRestTestStateCleaner.java    | 11 ++++---
 .../xpack/test/rest/XPackRestTestHelper.java  | 15 +++++----
 5 files changed, 44 insertions(+), 54 deletions(-)

diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java
index fc8e25e3ccc..20009dba41c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -67,12 +68,14 @@ public class StartBasicLicenseTests extends AbstractLicensesIntegrationTestCase
         }
 
         RestClient restClient = getRestClient();
-        Response response = restClient.performRequest("GET", "/_xpack/license/basic_status");
+        Response response = restClient.performRequest(new Request("GET", "/_xpack/license/basic_status"));
         String body = Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(200, response.getStatusLine().getStatusCode());
         assertEquals("{\"eligible_to_start_basic\":true}", body);
 
-        Response response2 = restClient.performRequest("POST", "/_xpack/license/start_basic?acknowledge=true");
+        Request ackRequest = new Request("POST", "/_xpack/license/start_basic");
+        ackRequest.addParameter("acknowledge", "true");
+        Response response2 = restClient.performRequest(ackRequest);
         String body2 = Streams.copyToString(new InputStreamReader(response2.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(200, response2.getStatusLine().getStatusCode());
         assertTrue(body2.contains("\"acknowledged\":true"));
@@ -86,20 +89,19 @@ public class StartBasicLicenseTests extends AbstractLicensesIntegrationTestCase
         long expirationMillis = licensingClient.prepareGetLicense().get().license().expiryDate();
         assertEquals(LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS, expirationMillis);
 
-        Response response3 = restClient.performRequest("GET", "/_xpack/license");
+        Response response3 = restClient.performRequest(new Request("GET", "/_xpack/license"));
         String body3 = Streams.copyToString(new InputStreamReader(response3.getEntity().getContent(), StandardCharsets.UTF_8));
         assertTrue(body3.contains("\"type\" : \"basic\""));
         assertFalse(body3.contains("expiry_date"));
         assertFalse(body3.contains("expiry_date_in_millis"));
 
-
-        Response response4 = restClient.performRequest("GET", "/_xpack/license/basic_status");
+        Response response4 = restClient.performRequest(new Request("GET", "/_xpack/license/basic_status"));
         String body4 = Streams.copyToString(new InputStreamReader(response4.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(200, response3.getStatusLine().getStatusCode());
         assertEquals("{\"eligible_to_start_basic\":false}", body4);
 
         ResponseException ex = expectThrows(ResponseException.class,
-                () -> restClient.performRequest("POST", "/_xpack/license/start_basic"));
+                () -> restClient.performRequest(new Request("POST", "/_xpack/license/start_basic")));
         Response response5 = ex.getResponse();
         String body5 = Streams.copyToString(new InputStreamReader(response5.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(403, response5.getStatusLine().getStatusCode());
@@ -118,7 +120,7 @@ public class StartBasicLicenseTests extends AbstractLicensesIntegrationTestCase
             assertEquals("trial", getLicenseResponse.license().type());
         });
 
-        Response response2 = getRestClient().performRequest("POST", "/_xpack/license/start_basic");
+        Response response2 = getRestClient().performRequest(new Request("POST", "/_xpack/license/start_basic"));
         String body2 = Streams.copyToString(new InputStreamReader(response2.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(200, response2.getStatusLine().getStatusCode());
         assertTrue(body2.contains("\"acknowledged\":false"));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java
index 24ba4bd2bd6..9c07965a7ec 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -54,13 +55,13 @@ public class StartTrialLicenseTests extends AbstractLicensesIntegrationTestCase
         ensureStartingWithBasic();
 
         RestClient restClient = getRestClient();
-        Response response = restClient.performRequest("GET", "/_xpack/license/trial_status");
+        Response response = restClient.performRequest(new Request("GET", "/_xpack/license/trial_status"));
         String body = Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(200, response.getStatusLine().getStatusCode());
         assertEquals("{\"eligible_to_start_trial\":true}", body);
 
         // Test that starting will fail without acknowledgement
-        Response response2 = restClient.performRequest("POST", "/_xpack/license/start_trial");
+        Response response2 = restClient.performRequest(new Request("POST", "/_xpack/license/start_trial"));
         String body2 = Streams.copyToString(new InputStreamReader(response2.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(200, response2.getStatusLine().getStatusCode());
         assertTrue(body2.contains("\"trial_was_started\":false"));
@@ -74,7 +75,10 @@ public class StartTrialLicenseTests extends AbstractLicensesIntegrationTestCase
 
         String type = randomFrom(LicenseService.VALID_TRIAL_TYPES);
 
-        Response response3 = restClient.performRequest("POST", "/_xpack/license/start_trial?acknowledge=true&type=" + type);
+        Request ackRequest = new Request("POST", "/_xpack/license/start_trial");
+        ackRequest.addParameter("acknowledge", "true");
+        ackRequest.addParameter("type", type);
+        Response response3 = restClient.performRequest(ackRequest);
         String body3 = Streams.copyToString(new InputStreamReader(response3.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(200, response3.getStatusLine().getStatusCode());
         assertTrue(body3.contains("\"trial_was_started\":true"));
@@ -86,15 +90,17 @@ public class StartTrialLicenseTests extends AbstractLicensesIntegrationTestCase
             assertEquals(type, postTrialLicenseResponse.license().type());
         });
 
-        Response response4 = restClient.performRequest("GET", "/_xpack/license/trial_status");
+        Response response4 = restClient.performRequest(new Request("GET", "/_xpack/license/trial_status"));
         String body4 = Streams.copyToString(new InputStreamReader(response4.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(200, response4.getStatusLine().getStatusCode());
         assertEquals("{\"eligible_to_start_trial\":false}", body4);
 
         String secondAttemptType = randomFrom(LicenseService.VALID_TRIAL_TYPES);
 
-        ResponseException ex = expectThrows(ResponseException.class,
-                () -> restClient.performRequest("POST", "/_xpack/license/start_trial?acknowledge=true&type=" + secondAttemptType));
+        Request startTrialWhenStartedRequest = new Request("POST", "/_xpack/license/start_trial");
+        startTrialWhenStartedRequest.addParameter("acknowledge", "true");
+        startTrialWhenStartedRequest.addParameter("type", secondAttemptType);
+        ResponseException ex = expectThrows(ResponseException.class, () -> restClient.performRequest(startTrialWhenStartedRequest));
         Response response5 = ex.getResponse();
         String body5 = Streams.copyToString(new InputStreamReader(response5.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(403, response5.getStatusLine().getStatusCode());
@@ -105,8 +111,9 @@ public class StartTrialLicenseTests extends AbstractLicensesIntegrationTestCase
     public void testInvalidType() throws Exception {
         ensureStartingWithBasic();
 
-        ResponseException ex = expectThrows(ResponseException.class, () ->
-                getRestClient().performRequest("POST", "/_xpack/license/start_trial?type=basic"));
+        Request request = new Request("POST", "/_xpack/license/start_trial");
+        request.addParameter("type", "basic");
+        ResponseException ex = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
         Response response = ex.getResponse();
         String body = Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8));
         assertEquals(400, response.getStatusLine().getStatusCode());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
index 3e0d2d80651..50500f0e939 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
@@ -44,20 +44,11 @@ public class MlRestTestStateCleaner {
         }
 
         try {
-            int statusCode = adminClient.performRequest("POST", "/_xpack/ml/datafeeds/_all/_stop")
-                    .getStatusLine().getStatusCode();
-            if (statusCode != 200) {
-                logger.error("Got status code " + statusCode + " when stopping datafeeds");
-            }
+            adminClient.performRequest(new Request("POST", "/_xpack/ml/datafeeds/_all/_stop"));
         } catch (Exception e1) {
             logger.warn("failed to stop all datafeeds. Forcing stop", e1);
             try {
-                int statusCode = adminClient
-                        .performRequest("POST", "/_xpack/ml/datafeeds/_all/_stop?force=true")
-                        .getStatusLine().getStatusCode();
-                if (statusCode != 200) {
-                    logger.error("Got status code " + statusCode + " when stopping datafeeds");
-                }
+                adminClient.performRequest(new Request("POST", "/_xpack/ml/datafeeds/_all/_stop?force=true"));
             } catch (Exception e2) {
                 logger.warn("Force-closing all data feeds failed", e2);
             }
@@ -67,10 +58,7 @@ public class MlRestTestStateCleaner {
 
         for (Map<String, Object> datafeed : datafeeds) {
             String datafeedId = (String) datafeed.get("datafeed_id");
-            int statusCode = adminClient.performRequest("DELETE", "/_xpack/ml/datafeeds/" + datafeedId).getStatusLine().getStatusCode();
-            if (statusCode != 200) {
-                logger.error("Got status code " + statusCode + " when deleting datafeed " + datafeedId);
-            }
+            adminClient.performRequest(new Request("DELETE", "/_xpack/ml/datafeeds/" + datafeedId));
         }
     }
 
@@ -86,17 +74,11 @@ public class MlRestTestStateCleaner {
         }
 
         try {
-            int statusCode = adminClient
-                    .performRequest("POST", "/_xpack/ml/anomaly_detectors/_all/_close")
-                    .getStatusLine().getStatusCode();
-            if (statusCode != 200) {
-                logger.error("Got status code " + statusCode + " when closing all jobs");
-            }
+            adminClient.performRequest(new Request("POST", "/_xpack/ml/anomaly_detectors/_all/_close"));
         } catch (Exception e1) {
             logger.warn("failed to close all jobs. Forcing closed", e1);
             try {
-                adminClient.performRequest("POST",
-                        "/_xpack/ml/anomaly_detectors/_all/_close?force=true");
+                adminClient.performRequest(new Request("POST", "/_xpack/ml/anomaly_detectors/_all/_close?force=true"));
             } catch (Exception e2) {
                 logger.warn("Force-closing all jobs failed", e2);
             }
@@ -106,10 +88,7 @@ public class MlRestTestStateCleaner {
 
         for (Map<String, Object> jobConfig : jobConfigs) {
             String jobId = (String) jobConfig.get("job_id");
-            int statusCode = adminClient.performRequest("DELETE", "/_xpack/ml/anomaly_detectors/" + jobId).getStatusLine().getStatusCode();
-            if (statusCode != 200) {
-                logger.error("Got status code " + statusCode + " when deleting job " + jobId);
-            }
+            adminClient.performRequest(new Request("DELETE", "/_xpack/ml/anomaly_detectors/" + jobId));
         }
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
index 49515973c3b..3eb9aa79b4b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.rollup;
 
 import org.apache.http.HttpStatus;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -17,7 +18,6 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -35,8 +35,9 @@ public class RollupRestTestStateCleaner {
     private static void waitForPendingTasks(RestClient adminClient) throws Exception {
         ESTestCase.assertBusy(() -> {
             try {
-                Response response = adminClient.performRequest("GET", "/_cat/tasks",
-                        Collections.singletonMap("detailed", "true"));
+                Request request = new Request("GET", "/_cat/tasks");
+                request.addParameter("detailed", "true");
+                Response response = adminClient.performRequest(request);
                 if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                     try (BufferedReader responseReader = new BufferedReader(
                             new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
@@ -63,7 +64,7 @@ public class RollupRestTestStateCleaner {
 
     @SuppressWarnings("unchecked")
     private static void deleteAllJobs(RestClient adminClient) throws Exception {
-        Response response = adminClient.performRequest("GET", "/_xpack/rollup/job/_all");
+        Response response = adminClient.performRequest(new Request("GET", "/_xpack/rollup/job/_all"));
         Map<String, Object> jobs = ESRestTestCase.entityAsMap(response);
         List<Map<String, Object>> jobConfigs =
                 (List<Map<String, Object>>) XContentMapValues.extractValue("jobs", jobs);
@@ -75,7 +76,7 @@ public class RollupRestTestStateCleaner {
         for (Map<String, Object> jobConfig : jobConfigs) {
             String jobId = (String) ((Map<String, Object>) jobConfig.get("config")).get("id");
             try {
-                response = adminClient.performRequest("DELETE", "/_xpack/rollup/job/" + jobId);
+                response = adminClient.performRequest(new Request("DELETE", "/_xpack/rollup/job/" + jobId));
             } catch (Exception e) {
                 // ok
             }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java
index 0e3627d64ff..5e9fd4a386b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java
@@ -10,6 +10,7 @@ import org.apache.http.HttpStatus;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -25,12 +26,10 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static java.util.Collections.singletonMap;
 import static org.junit.Assert.assertEquals;
 
 public final class XPackRestTestHelper {
@@ -47,8 +46,9 @@ public final class XPackRestTestHelper {
         ESTestCase.awaitBusy(() -> {
             String response;
             try {
-                response = EntityUtils
-                        .toString(client.performRequest("GET", "/_cat/nodes", singletonMap("h", "master,version")).getEntity());
+                Request request = new Request("GET", "/_cat/nodes");
+                request.addParameter("h", "master,version");
+                response = EntityUtils.toString(client.performRequest(request).getEntity());
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
@@ -67,7 +67,7 @@ public final class XPackRestTestHelper {
             ESTestCase.awaitBusy(() -> {
                 Map<?, ?> response;
                 try {
-                    String string = EntityUtils.toString(client.performRequest("GET", "/_template/" + template).getEntity());
+                    String string = EntityUtils.toString(client.performRequest(new Request("GET", "/_template/" + template)).getEntity());
                     response = XContentHelper.convertToMap(JsonXContent.jsonXContent, string, false);
                 } catch (ResponseException e) {
                     if (e.getResponse().getStatusLine().getStatusCode() == 404) {
@@ -89,8 +89,9 @@ public final class XPackRestTestHelper {
     public static void waitForPendingTasks(RestClient adminClient) throws Exception {
         ESTestCase.assertBusy(() -> {
             try {
-                Response response = adminClient.performRequest("GET", "/_cat/tasks",
-                        Collections.singletonMap("detailed", "true"));
+                Request request = new Request("GET", "/_cat/tasks");
+                request.addParameter("detailed", "true");
+                Response response = adminClient.performRequest(request);
                 // Check to see if there are tasks still active. We exclude the
                 // list tasks
                 // actions tasks form this otherwise we will always fail

From 1091ec115e59183e9d2f346b9db3c53700b81a3f Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 23 Jul 2018 11:28:24 -0700
Subject: [PATCH 139/260] specify subdirs of lib, bin, modules in package
 (#32253)

Explicitly include all subdirectories of these folders in
/usr/share/elasticsearch in package distributions so that they are
managed by the package manager. This change does really have an
effect in the 7.x series, where there are no subdirectories in bin, and
we were already doing this in lib and modules. It does have an effect in
the 6.x series where the bin/x-pack subdirectory was not previously
tracked by the package manager and could be left behind on removal in
rpm distributions.
---
 distribution/packages/build.gradle | 44 ++++++++++--------------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
index fcd69138da3..4fdaa5c1a01 100644
--- a/distribution/packages/build.gradle
+++ b/distribution/packages/build.gradle
@@ -125,32 +125,22 @@ Closure commonPackageConfig(String type, boolean oss) {
         fileMode 0644
       }
       into('lib') {
-        with copySpec {
-          with libFiles(oss)
-          // we need to specify every intermediate directory so we iterate through the parents; duplicate calls with the same part are fine
-          eachFile { FileCopyDetails fcp ->
-            String[] segments = fcp.relativePath.segments
-            for (int i = segments.length - 2; i > 0 && segments[i] != 'lib'; --i) {
-              directory('/' + segments[0..i].join('/'), 0755)
-            }
-            fcp.mode = 0644
-          }
-        }
+        with libFiles(oss)
       }
       into('modules') {
-        with copySpec {
-          with modulesFiles(oss)
-          // we need to specify every intermediate directory so we iterate through the parents; duplicate calls with the same part are fine
-          eachFile { FileCopyDetails fcp ->
-            String[] segments = fcp.relativePath.segments
-            for (int i = segments.length - 2; i > 0 && segments[i] != 'modules'; --i) {
-              directory('/' + segments[0..i].join('/'), 0755)
-            }
-            if (segments[-2] == 'bin') {
-              fcp.mode = 0755
-            } else {
-              fcp.mode = 0644
-            }
+        with modulesFiles(oss)
+      }
+      // we need to specify every intermediate directory in these paths so the package managers know they are explicitly
+      // intended to manage them; otherwise they may be left behind on uninstallation. duplicate calls of the same
+      // directory are fine
+      eachFile { FileCopyDetails fcp ->
+        String[] segments = fcp.relativePath.segments
+        for (int i = segments.length - 2; i > 2; --i) {
+          directory('/' + segments[0..i].join('/'), 0755)
+          if (segments[-2] == 'bin') {
+            fcp.mode = 0755
+          } else {
+            fcp.mode = 0644
           }
         }
       }
@@ -333,12 +323,6 @@ Closure commonRpmConfig(boolean oss) {
 
     // without this the rpm will have parent dirs of any files we copy in, eg /etc/elasticsearch
     addParentDirs false
-
-    // Declare the folders so that the RPM package manager removes
-    // them when upgrading or removing the package
-    directory('/usr/share/elasticsearch/bin', 0755)
-    directory('/usr/share/elasticsearch/lib', 0755)
-    directory('/usr/share/elasticsearch/modules', 0755)
   }
 }
 

From ac04ba42df10d1747b7ac2054bbea82c47914413 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 23 Jul 2018 11:45:04 -0700
Subject: [PATCH 140/260] [test] package pre-install java check (#32259)

This recreates a test that was added to the bats packaging tests
in #31343 but didn't make it over to the java project during when the
linux package tests were ported in #31943

When packages are installed but can not locate the java executable, they
should fail with a descriptive message
---
 .../packaging/test/ArchiveTestCase.java       |  2 +-
 .../packaging/test/PackageTestCase.java       | 19 +++++++++++++
 .../packaging/util/Packages.java              | 27 +++++++++++++++----
 3 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
index 20561115542..83edc8a0a93 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java
@@ -129,7 +129,7 @@ public abstract class ArchiveTestCase extends PackagingTestCase {
         });
 
         Platforms.onLinux(() -> {
-            final String javaPath = sh.run("which java").stdout.trim();
+            final String javaPath = sh.run("command -v java").stdout.trim();
 
             try {
                 sh.run("chmod -x '" + javaPath + "'");
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java
index 28a767e95ae..95705e94a43 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java
@@ -30,16 +30,20 @@ import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
 import static org.elasticsearch.packaging.util.FileUtils.assertPathsDontExist;
+import static org.elasticsearch.packaging.util.FileUtils.mv;
 import static org.elasticsearch.packaging.util.Packages.SYSTEMD_SERVICE;
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
 import static org.elasticsearch.packaging.util.Packages.install;
 import static org.elasticsearch.packaging.util.Packages.remove;
+import static org.elasticsearch.packaging.util.Packages.runInstallCommand;
 import static org.elasticsearch.packaging.util.Packages.startElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
 import static org.elasticsearch.packaging.util.Platforms.getOsRelease;
@@ -75,6 +79,21 @@ public abstract class PackageTestCase extends PackagingTestCase {
         assumeTrue("only compatible distributions", distribution().packaging.compatible);
     }
 
+    public void test05InstallFailsWhenJavaMissing() {
+        final Shell sh = new Shell();
+        final Result java = sh.run("command -v java");
+
+        final Path originalJavaPath = Paths.get(java.stdout.trim());
+        final Path relocatedJavaPath = originalJavaPath.getParent().resolve("java.relocated");
+        try {
+            mv(originalJavaPath, relocatedJavaPath);
+            final Result installResult = runInstallCommand(distribution());
+            assertThat(installResult.exitCode, is(1));
+        } finally {
+            mv(relocatedJavaPath, originalJavaPath);
+        }
+    }
+
     public void test10InstallPackage() {
         assertRemoved(distribution());
         installation = install(distribution());
diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java
index 6e80d9e027d..be7edc5e8f9 100644
--- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java
+++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Packages.java
@@ -67,7 +67,10 @@ public class Packages {
         Platforms.onDPKG(() -> {
             assertThat(status.exitCode, anyOf(is(0), is(1)));
             if (status.exitCode == 0) {
-                assertTrue(Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find());
+                assertTrue("an uninstalled status should be indicated: " + status.stdout,
+                    Pattern.compile("(?m)^Status:.+deinstall ok").matcher(status.stdout).find() ||
+                    Pattern.compile("(?m)^Status:.+ok not-installed").matcher(status.stdout).find()
+                );
             }
         });
     }
@@ -90,13 +93,27 @@ public class Packages {
     }
 
     public static Installation install(Distribution distribution, String version) {
+        final Result result = runInstallCommand(distribution, version);
+        if (result.exitCode != 0) {
+            throw new RuntimeException("Installing distribution " + distribution + " version " + version + " failed: " + result);
+        }
+
+        return Installation.ofPackage(distribution.packaging);
+    }
+
+    public static Result runInstallCommand(Distribution distribution) {
+        return runInstallCommand(distribution, getCurrentVersion());
+    }
+
+    public static Result runInstallCommand(Distribution distribution, String version) {
         final Shell sh = new Shell();
         final Path distributionFile = getDistributionFile(distribution, version);
 
-        Platforms.onRPM(() -> sh.run("rpm -i " + distributionFile));
-        Platforms.onDPKG(() -> sh.run("dpkg -i " + distributionFile));
-
-        return Installation.ofPackage(distribution.packaging);
+        if (Platforms.isRPM()) {
+            return sh.runIgnoreExitCode("rpm -i " + distributionFile);
+        } else {
+            return sh.runIgnoreExitCode("dpkg -i " + distributionFile);
+        }
     }
 
     public static void remove(Distribution distribution) {

From 0b72132553a4f7bd29cb6fd6b7fffd860d431874 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 Jul 2018 14:52:23 -0400
Subject: [PATCH 141/260] Silence SSL reload test that fails on JDK 11

This commit adds an assumption for an SSL reload test that we are not on
JDK 11 as the test currently fails there.

Relates #32293
---
 .../java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
index 03f963cc59c..0090c58ce0c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ssl;
 
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -92,6 +93,7 @@ public class SSLReloadIntegTests extends SecurityIntegTestCase {
     }
 
     public void testThatSSLConfigurationReloadsOnModification() throws Exception {
+        assumeFalse("test fails on JDK 11 currently", JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0);
         Path keyPath = createTempDir().resolve("testnode_updated.pem");
         Path certPath = createTempDir().resolve("testnode_updated.crt");
         Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem"), keyPath);

From bba1da642d9d8faddcf356af39ffb75f6ad6c145 Mon Sep 17 00:00:00 2001
From: James Baiera <james.baiera@gmail.com>
Date: Mon, 23 Jul 2018 15:11:41 -0400
Subject: [PATCH 142/260] Add new permission for JDK11 to load JAAS libraries
 (#32132)

Hadoop's security model uses the OS level authentication modules to collect
information about the current user. In JDK 11, the UnixLoginModule makes
use of a new permission to determine if the executing code is allowed to load
the libraries required to pull the user information from the OS. This PR adds
that permission and re-enables the tests that were previously failing when
testing against JDK 11.
---
 plugins/repository-hdfs/build.gradle          | 21 +------------------
 .../plugin-metadata/plugin-security.policy    |  1 +
 2 files changed, 2 insertions(+), 20 deletions(-)

diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index b3ae34012aa..8856ae1526a 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -1,4 +1,4 @@
-1/*
+/*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
  * this work for additional information regarding copyright
@@ -214,25 +214,6 @@ RestIntegTestTask integTestSecureHa = project.tasks.create('integTestSecureHa',
   description = "Runs rest tests against an elasticsearch cluster with HDFS configured with HA Namenode and secured by MIT Kerberos."
 }
 
-if (rootProject.ext.compilerJavaVersion.isJava11()) {
-  // TODO remove when: https://github.com/elastic/elasticsearch/issues/31498
-  integTestRunner {
-    systemProperty 'tests.rest.blacklist', [
-            'hdfs_repository/30_snapshot/take snapshot',
-            'hdfs_repository/40_restore/Create a snapshot and then restore it',
-            'hdfs_repository/20_repository_verify/HDFS Repository Verify',
-            'hdfs_repository/30_snapshot_get/Get a snapshot',
-            'hdfs_repository/20_repository_create/HDFS Repository Creation',
-            'hdfs_repository/20_repository_delete/HDFS Delete Repository',
-            'hdfs_repository/30_snapshot_readonly/Get a snapshot - readonly',
-    ].join(',')
-  }
-}
-if (rootProject.ext.runtimeJavaVersion.isJava11() || rootProject.ext.compilerJavaVersion.isJava11()) {
-  // TODO remove when: https://github.com/elastic/elasticsearch/issues/31498
-  integTestHa.enabled = false
-}
-
 // Determine HDFS Fixture compatibility for the current build environment.
 boolean fixtureSupported = false
 if (Os.isFamily(Os.FAMILY_WINDOWS)) {
diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
index f6476f290bc..897596bbd85 100644
--- a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
@@ -61,6 +61,7 @@ grant {
 
   // Hadoop depends on OS level user information for simple authentication
   // Unix: UnixLoginModule: com.sun.security.auth.module.UnixSystem.UnixSystem init
+  permission java.lang.RuntimePermission "loadLibrary.jaas";
   permission java.lang.RuntimePermission "loadLibrary.jaas_unix";
   // Windows: NTLoginModule: com.sun.security.auth.module.NTSystem.loadNative
   permission java.lang.RuntimePermission "loadLibrary.jaas_nt";

From 1b1aa4ecff2a7d4ac4a37be77226a444259aec95 Mon Sep 17 00:00:00 2001
From: Julie Tibshirani <julie.tibshirani@elastic.co>
Date: Mon, 23 Jul 2018 12:25:42 -0700
Subject: [PATCH 143/260] Fix a test bug around nested aggregations and field
 aliases. (#32287)

This issue affected both NestedAggregatorTest and ReverseNestedAggregatorTest.
---
 .../aggregations/bucket/nested/NestedAggregatorTests.java | 5 +++--
 .../bucket/nested/ReverseNestedAggregatorTests.java       | 8 ++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
index c893e595964..e1206cb8d15 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
@@ -655,6 +655,7 @@ public class NestedAggregatorTests extends AggregatorTestCase {
 
     public void testFieldAlias() throws IOException {
         int numRootDocs = randomIntBetween(1, 20);
+        int expectedNestedDocs = 0;
 
         MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
             NumberFieldMapper.NumberType.LONG);
@@ -665,6 +666,7 @@ public class NestedAggregatorTests extends AggregatorTestCase {
                 for (int i = 0; i < numRootDocs; i++) {
                     List<Document> documents = new ArrayList<>();
                     int numNestedDocs = randomIntBetween(0, 20);
+                    expectedNestedDocs += numNestedDocs;
                     generateDocuments(documents, numNestedDocs, i, NESTED_OBJECT, VALUE_FIELD_NAME);
 
                     Document document = new Document();
@@ -681,7 +683,6 @@ public class NestedAggregatorTests extends AggregatorTestCase {
             try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) {
                 NestedAggregationBuilder agg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation(
                     max(MAX_AGG_NAME).field(VALUE_FIELD_NAME));
-
                 NestedAggregationBuilder aliasAgg = nested(NESTED_AGG, NESTED_OBJECT).subAggregation(
                     max(MAX_AGG_NAME).field(VALUE_FIELD_NAME + "-alias"));
 
@@ -690,8 +691,8 @@ public class NestedAggregatorTests extends AggregatorTestCase {
                 Nested aliasNested = search(newSearcher(indexReader, false, true),
                     new MatchAllDocsQuery(), aliasAgg, fieldType);
 
-                assertTrue(nested.getDocCount() > 0);
                 assertEquals(nested, aliasNested);
+                assertEquals(expectedNestedDocs, nested.getDocCount());
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
index 99322af2264..fd831e5076c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java
@@ -169,6 +169,7 @@ public class ReverseNestedAggregatorTests extends AggregatorTestCase {
 
     public void testFieldAlias() throws IOException {
         int numParentDocs = randomIntBetween(1, 20);
+        int expectedParentDocs = 0;
 
         MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
             NumberFieldMapper.NumberType.LONG);
@@ -179,6 +180,10 @@ public class ReverseNestedAggregatorTests extends AggregatorTestCase {
                 for (int i = 0; i < numParentDocs; i++) {
                     List<Document> documents = new ArrayList<>();
                     int numNestedDocs = randomIntBetween(0, 20);
+                    if (numNestedDocs > 0) {
+                        expectedParentDocs++;
+                    }
+
                     for (int nested = 0; nested < numNestedDocs; nested++) {
                         Document document = new Document();
                         document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(i)),
@@ -203,7 +208,6 @@ public class ReverseNestedAggregatorTests extends AggregatorTestCase {
             }
 
             try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) {
-
                 MaxAggregationBuilder maxAgg = max(MAX_AGG_NAME).field(VALUE_FIELD_NAME);
                 MaxAggregationBuilder aliasMaxAgg = max(MAX_AGG_NAME).field(VALUE_FIELD_NAME + "-alias");
 
@@ -220,8 +224,8 @@ public class ReverseNestedAggregatorTests extends AggregatorTestCase {
                 ReverseNested reverseNested = nested.getAggregations().get(REVERSE_AGG_NAME);
                 ReverseNested aliasReverseNested = aliasNested.getAggregations().get(REVERSE_AGG_NAME);
 
-                assertTrue(reverseNested.getDocCount() > 0);
                 assertEquals(reverseNested, aliasReverseNested);
+                assertEquals(expectedParentDocs, reverseNested.getDocCount());
             }
         }
     }

From fe6bb75eb49ad66ca7a6885cc0a80467d44d9545 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Mon, 23 Jul 2018 22:25:02 +0200
Subject: [PATCH 144/260] Rename ranking evaluation `quality_level` to
 `metric_score` (#32168)

The notion of "quality" is an overloaded term in the search ranking evaluation
context. Its usually used to decribe certain levels of "good" vs. "bad" of a
seach result with respect to the users information need. We currently report the
result of the ranking evaluation as `quality_level` which is a bit missleading.
This changes the response parameter name to `metric_score` which fits better.
---
 .../org/elasticsearch/client/RankEvalIT.java  |  2 +-
 .../documentation/SearchDocumentationIT.java  |  4 +--
 docs/reference/search/rank-eval.asciidoc      |  6 ++--
 .../rankeval/DiscountedCumulativeGain.java    |  4 +--
 .../index/rankeval/EvalQueryQuality.java      | 26 ++++++++---------
 .../index/rankeval/EvaluationMetric.java      | 25 ++++++++---------
 .../index/rankeval/MeanReciprocalRank.java    |  3 +-
 .../index/rankeval/RankEvalResponse.java      | 19 ++++++-------
 .../DiscountedCumulativeGainTests.java        | 16 +++++------
 .../index/rankeval/EvalQueryQualityTests.java |  6 ++--
 .../rankeval/ExpectedReciprocalRankTests.java | 10 +++----
 .../rankeval/MeanReciprocalRankTests.java     | 14 +++++-----
 .../index/rankeval/PrecisionAtKTests.java     | 18 ++++++------
 .../index/rankeval/RankEvalRequestIT.java     | 12 ++++----
 .../index/rankeval/RankEvalResponseTests.java |  8 +++---
 .../rest-api-spec/test/rank_eval/10_basic.yml | 22 +++++++--------
 .../rest-api-spec/test/rank_eval/20_dcg.yml   | 28 +++++++++----------
 .../test/rank_eval/30_failures.yml            |  4 +--
 .../test/rank_eval/40_rank_eval_templated.yml |  2 +-
 .../test/rankeval/10_rankeval.yml             |  2 +-
 20 files changed, 114 insertions(+), 117 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
index 2890257b236..330afafd9ef 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
@@ -80,7 +80,7 @@ public class RankEvalIT extends ESRestHighLevelClientTestCase {
         RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
         // the expected Prec@ for the first query is 5/7 and the expected Prec@ for the second is 1/7, divided by 2 to get the average
         double expectedPrecision = (1.0 / 7.0 + 5.0 / 7.0) / 2.0;
-        assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
+        assertEquals(expectedPrecision, response.getMetricScore(), Double.MIN_VALUE);
         Map<String, EvalQueryQuality> partialResults = response.getPartialResults();
         assertEquals(2, partialResults.size());
         EvalQueryQuality amsterdamQueryQuality = partialResults.get("amsterdam_query");
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
index c60f2d4c92b..2f743c786ba 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
@@ -1136,14 +1136,14 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase {
             // end::rank-eval-execute
 
             // tag::rank-eval-response
-            double evaluationResult = response.getEvaluationResult();   // <1>
+            double evaluationResult = response.getMetricScore();   // <1>
             assertEquals(1.0 / 3.0, evaluationResult, 0.0);
             Map<String, EvalQueryQuality> partialResults =
                     response.getPartialResults();
             EvalQueryQuality evalQuality =
                     partialResults.get("kimchy_query");                 // <2>
             assertEquals("kimchy_query", evalQuality.getId());
-            double qualityLevel = evalQuality.getQualityLevel();        // <3>
+            double qualityLevel = evalQuality.metricScore();        // <3>
             assertEquals(1.0 / 3.0, qualityLevel, 0.0);
             List<RatedSearchHit> hitsAndRatings = evalQuality.getHitsAndRatings();
             RatedSearchHit ratedSearchHit = hitsAndRatings.get(2);
diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
index cf13b9f7b06..ef715dfca8c 100644
--- a/docs/reference/search/rank-eval.asciidoc
+++ b/docs/reference/search/rank-eval.asciidoc
@@ -270,10 +270,10 @@ that shows potential errors of individual queries. The response has the followin
 --------------------------------
 {
     "rank_eval": {
-        "quality_level": 0.4, <1>
+        "metric_score": 0.4, <1>
         "details": {  
             "my_query_id1": { <2>
-                "quality_level": 0.6, <3>
+                "metric_score": 0.6, <3>
                 "unrated_docs": [ <4>
                     {
                         "_index": "my_index",
@@ -308,7 +308,7 @@ that shows potential errors of individual queries. The response has the followin
 
 <1> the overall evaluation quality calculated by the defined metric
 <2> the `details` section contains one entry for every query in the original `requests` section, keyed by the search request id
-<3> the `quality_level` in the `details` section shows the contribution of this query to the global quality score
+<3> the `metric_score` in the `details` section shows the contribution of this query to the global quality metric score
 <4> the `unrated_docs` section contains an `_index` and `_id` entry for each document in the search result for this
 query that didn't have a ratings value. This can be used to ask the user to supply ratings for these documents
 <5> the `hits` section shows a grouping of the search results with their supplied rating
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java
index cab32377323..a6a6830a99c 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java
@@ -126,8 +126,6 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
     @Override
     public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
             List<RatedDocument> ratedDocs) {
-        List<Integer> allRatings = ratedDocs.stream().mapToInt(RatedDocument::getRating).boxed()
-                .collect(Collectors.toList());
         List<RatedSearchHit> ratedHits = joinHitsWithRatings(hits, ratedDocs);
         List<Integer> ratingsInSearchHits = new ArrayList<>(ratedHits.size());
         int unratedResults = 0;
@@ -144,6 +142,8 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
         double idcg = 0;
 
         if (normalize) {
+            List<Integer> allRatings = ratedDocs.stream().mapToInt(RatedDocument::getRating).boxed()
+                    .collect(Collectors.toList());
             Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder()));
             idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size())));
             if (idcg != 0) {
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
index 91ba1ce6169..f065a34787c 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
@@ -41,19 +41,19 @@ import java.util.Objects;
 public class EvalQueryQuality implements ToXContentFragment, Writeable {
 
     private final String queryId;
-    private final double evaluationResult;
+    private final double metricScore;
     private MetricDetail optionalMetricDetails;
     private final List<RatedSearchHit> ratedHits;
 
-    public EvalQueryQuality(String id, double evaluationResult) {
+    public EvalQueryQuality(String id, double metricScore) {
         this.queryId = id;
-        this.evaluationResult = evaluationResult;
+        this.metricScore = metricScore;
         this.ratedHits = new ArrayList<>();
     }
 
     public EvalQueryQuality(StreamInput in) throws IOException {
         this.queryId = in.readString();
-        this.evaluationResult = in.readDouble();
+        this.metricScore = in.readDouble();
         this.ratedHits = in.readList(RatedSearchHit::new);
         this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetail.class);
     }
@@ -61,7 +61,7 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
     // only used for parsing internally
     private EvalQueryQuality(String queryId, ParsedEvalQueryQuality builder) {
         this.queryId = queryId;
-        this.evaluationResult = builder.evaluationResult;
+        this.metricScore = builder.evaluationResult;
         this.optionalMetricDetails = builder.optionalMetricDetails;
         this.ratedHits = builder.ratedHits;
     }
@@ -69,7 +69,7 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(queryId);
-        out.writeDouble(evaluationResult);
+        out.writeDouble(metricScore);
         out.writeList(ratedHits);
         out.writeOptionalNamedWriteable(this.optionalMetricDetails);
     }
@@ -78,8 +78,8 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
         return queryId;
     }
 
-    public double getQualityLevel() {
-        return evaluationResult;
+    public double metricScore() {
+        return metricScore;
     }
 
     public void setMetricDetails(MetricDetail breakdown) {
@@ -101,7 +101,7 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(queryId);
-        builder.field(QUALITY_LEVEL_FIELD.getPreferredName(), this.evaluationResult);
+        builder.field(METRIC_SCORE_FIELD.getPreferredName(), this.metricScore);
         builder.startArray(UNRATED_DOCS_FIELD.getPreferredName());
         for (DocumentKey key : EvaluationMetric.filterUnratedDocuments(ratedHits)) {
             builder.startObject();
@@ -122,7 +122,7 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
         return builder;
     }
 
-    private static final ParseField QUALITY_LEVEL_FIELD = new ParseField("quality_level");
+    static final ParseField METRIC_SCORE_FIELD = new ParseField("metric_score");
     private static final ParseField UNRATED_DOCS_FIELD = new ParseField("unrated_docs");
     private static final ParseField HITS_FIELD = new ParseField("hits");
     private static final ParseField METRIC_DETAILS_FIELD = new ParseField("metric_details");
@@ -136,7 +136,7 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
     }
 
     static {
-        PARSER.declareDouble((obj, value) -> obj.evaluationResult = value, QUALITY_LEVEL_FIELD);
+        PARSER.declareDouble((obj, value) -> obj.evaluationResult = value, METRIC_SCORE_FIELD);
         PARSER.declareObject((obj, value) -> obj.optionalMetricDetails = value, (p, c) -> parseMetricDetail(p),
                 METRIC_DETAILS_FIELD);
         PARSER.declareObjectArray((obj, list) -> obj.ratedHits = list, (p, c) -> RatedSearchHit.parse(p), HITS_FIELD);
@@ -164,13 +164,13 @@ public class EvalQueryQuality implements ToXContentFragment, Writeable {
         }
         EvalQueryQuality other = (EvalQueryQuality) obj;
         return Objects.equals(queryId, other.queryId) &&
-                Objects.equals(evaluationResult, other.evaluationResult) &&
+                Objects.equals(metricScore, other.metricScore) &&
                 Objects.equals(ratedHits, other.ratedHits) &&
                 Objects.equals(optionalMetricDetails, other.optionalMetricDetails);
     }
 
     @Override
     public final int hashCode() {
-        return Objects.hash(queryId, evaluationResult, ratedHits, optionalMetricDetails);
+        return Objects.hash(queryId, metricScore, ratedHits, optionalMetricDetails);
     }
 }
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
index 37898fd9516..d1e89890477 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
@@ -39,23 +39,22 @@ import java.util.stream.Collectors;
 public interface EvaluationMetric extends ToXContentObject, NamedWriteable {
 
     /**
-     * Returns a single metric representing the ranking quality of a set of returned
-     * documents wrt. to a set of document ids labeled as relevant for this search.
+     * Evaluates a single ranking evaluation case.
      *
      * @param taskId
-     *            the id of the query for which the ranking is currently evaluated
+     *            an identifier of the query for which the search ranking is
+     *            evaluated
      * @param hits
-     *            the result hits as returned by a search request
+     *            the search result hits
      * @param ratedDocs
-     *            the documents that were ranked by human annotators for this query
-     *            case
-     * @return some metric representing the quality of the result hit list wrt. to
-     *         relevant doc ids.
+     *            the documents that contain the document rating for this query case
+     * @return an {@link EvalQueryQuality} instance that contains the metric score
+     *         with respect to the provided search hits and ratings
      */
     EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List<RatedDocument> ratedDocs);
 
     /**
-     * join hits with rated documents using the joint _index/_id document key
+     * Joins hits with rated documents using the joint _index/_id document key.
      */
     static List<RatedSearchHit> joinHitsWithRatings(SearchHit[] hits, List<RatedDocument> ratedDocs) {
         Map<DocumentKey, RatedDocument> ratedDocumentMap = ratedDocs.stream()
@@ -74,7 +73,7 @@ public interface EvaluationMetric extends ToXContentObject, NamedWriteable {
     }
 
     /**
-     * filter @link {@link RatedSearchHit} that don't have a rating
+     * Filter {@link RatedSearchHit}s that do not have a rating.
      */
     static List<DocumentKey> filterUnratedDocuments(List<RatedSearchHit> ratedHits) {
         return ratedHits.stream().filter(hit -> hit.getRating().isPresent() == false)
@@ -82,11 +81,11 @@ public interface EvaluationMetric extends ToXContentObject, NamedWriteable {
     }
 
     /**
-     * how evaluation metrics for particular search queries get combined for the overall evaluation score.
-     * Defaults to averaging over the partial results.
+     * Combine several {@link EvalQueryQuality} results into the overall evaluation score.
+     * This defaults to averaging over the partial results, but can be overwritten to obtain a different behavior.
      */
     default double combine(Collection<EvalQueryQuality> partialResults) {
-        return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size();
+        return partialResults.stream().mapToDouble(EvalQueryQuality::metricScore).sum() / partialResults.size();
     }
 
     /**
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java
index eb20dc8c680..5781f13dafe 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java
@@ -110,8 +110,7 @@ public class MeanReciprocalRank implements EvaluationMetric {
      * Compute ReciprocalRank based on provided relevant document IDs.
      **/
     @Override
-    public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
-            List<RatedDocument> ratedDocs) {
+    public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List<RatedDocument> ratedDocs) {
         List<RatedSearchHit> ratedHits = joinHitsWithRatings(hits, ratedDocs);
         int firstRelevant = -1;
         int rank = 1;
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java
index 6dd3c1338fa..6efff154b62 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java
@@ -48,15 +48,15 @@ import java.util.stream.Collectors;
 public class RankEvalResponse extends ActionResponse implements ToXContentObject {
 
     /** The overall evaluation result. */
-    private double evaluationResult;
+    private double metricScore;
     /** details about individual ranking evaluation queries, keyed by their id */
     private Map<String, EvalQueryQuality> details;
     /** exceptions for specific ranking evaluation queries, keyed by their id */
     private Map<String, Exception> failures;
 
-    public RankEvalResponse(double qualityLevel, Map<String, EvalQueryQuality> partialResults,
+    public RankEvalResponse(double metricScore, Map<String, EvalQueryQuality> partialResults,
             Map<String, Exception> failures) {
-        this.evaluationResult = qualityLevel;
+        this.metricScore = metricScore;
         this.details =  new HashMap<>(partialResults);
         this.failures = new HashMap<>(failures);
     }
@@ -65,8 +65,8 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject
         // only used in RankEvalAction#newResponse()
     }
 
-    public double getEvaluationResult() {
-        return evaluationResult;
+    public double getMetricScore() {
+        return metricScore;
     }
 
     public Map<String, EvalQueryQuality> getPartialResults() {
@@ -85,7 +85,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeDouble(evaluationResult);
+        out.writeDouble(metricScore);
         out.writeVInt(details.size());
         for (String queryId : details.keySet()) {
             out.writeString(queryId);
@@ -101,7 +101,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        this.evaluationResult = in.readDouble();
+        this.metricScore = in.readDouble();
         int partialResultSize = in.readVInt();
         this.details = new HashMap<>(partialResultSize);
         for (int i = 0; i < partialResultSize; i++) {
@@ -120,7 +120,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field("quality_level", evaluationResult);
+        builder.field("metric_score", metricScore);
         builder.startObject("details");
         for (String key : details.keySet()) {
             details.get(key).toXContent(builder, params);
@@ -137,7 +137,6 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject
         return builder;
     }
 
-    private static final ParseField QUALITY_LEVEL_FIELD = new ParseField("quality_level");
     private static final ParseField DETAILS_FIELD = new ParseField("details");
     private static final ParseField FAILURES_FIELD = new ParseField("failures");
     @SuppressWarnings("unchecked")
@@ -147,7 +146,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject
                     ((List<EvalQueryQuality>) a[1]).stream().collect(Collectors.toMap(EvalQueryQuality::getId, Function.identity())),
                     ((List<Tuple<String, Exception>>) a[2]).stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2))));
     static {
-        PARSER.declareDouble(ConstructingObjectParser.constructorArg(), QUALITY_LEVEL_FIELD);
+        PARSER.declareDouble(ConstructingObjectParser.constructorArg(), EvalQueryQuality.METRIC_SCORE_FIELD);
         PARSER.declareNamedObjects(ConstructingObjectParser.optionalConstructorArg(), (p, c, n) -> EvalQueryQuality.fromXContent(p, n),
                 DETAILS_FIELD);
         PARSER.declareNamedObjects(ConstructingObjectParser.optionalConstructorArg(), (p, c, n) -> {
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
index e768c297333..468a1ac2e57 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
@@ -76,7 +76,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
             hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
         }
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
-        assertEquals(EXPECTED_DCG, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
+        assertEquals(EXPECTED_DCG, dcg.evaluate("id", hits, rated).metricScore(), DELTA);
 
         /**
          * Check with normalization: to get the maximal possible dcg, sort documents by
@@ -94,7 +94,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
          * idcg = 14.595390756454922 (sum of last column)
          */
         dcg = new DiscountedCumulativeGain(true, null, 10);
-        assertEquals(EXPECTED_NDCG, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
+        assertEquals(EXPECTED_NDCG, dcg.evaluate("id", hits, rated).metricScore(), DELTA);
     }
 
     /**
@@ -127,7 +127,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         }
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         EvalQueryQuality result = dcg.evaluate("id", hits, rated);
-        assertEquals(12.779642067948913, result.getQualityLevel(), DELTA);
+        assertEquals(12.779642067948913, result.metricScore(), DELTA);
         assertEquals(2, filterUnratedDocuments(result.getHitsAndRatings()).size());
 
         /**
@@ -146,7 +146,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
          * idcg = 13.347184833073591 (sum of last column)
          */
         dcg = new DiscountedCumulativeGain(true, null, 10);
-        assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
+        assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).metricScore(), DELTA);
     }
 
     /**
@@ -184,7 +184,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         }
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs);
-        assertEquals(12.392789260714371, result.getQualityLevel(), DELTA);
+        assertEquals(12.392789260714371, result.metricScore(), DELTA);
         assertEquals(1, filterUnratedDocuments(result.getHitsAndRatings()).size());
 
         /**
@@ -204,7 +204,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
          * idcg = 13.347184833073591 (sum of last column)
          */
         dcg = new DiscountedCumulativeGain(true, null, 10);
-        assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), DELTA);
+        assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).metricScore(), DELTA);
     }
 
     /**
@@ -223,13 +223,13 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         SearchHit[] hits = new SearchHit[0];
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs);
-        assertEquals(0.0d, result.getQualityLevel(), DELTA);
+        assertEquals(0.0d, result.metricScore(), DELTA);
         assertEquals(0, filterUnratedDocuments(result.getHitsAndRatings()).size());
 
         // also check normalized
         dcg = new DiscountedCumulativeGain(true, null, 10);
         result = dcg.evaluate("id", hits, ratedDocs);
-        assertEquals(0.0d, result.getQualityLevel(), DELTA);
+        assertEquals(0.0d, result.metricScore(), DELTA);
         assertEquals(0, filterUnratedDocuments(result.getHitsAndRatings()).size());
     }
 
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
index c9251bb8090..7424542ac26 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
@@ -129,7 +129,7 @@ public class EvalQueryQualityTests extends ESTestCase {
 
     private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) {
         String id = original.getId();
-        double qualityLevel = original.getQualityLevel();
+        double metricScore = original.metricScore();
         List<RatedSearchHit> ratedHits = new ArrayList<>(original.getHitsAndRatings());
         MetricDetail metricDetails = original.getMetricDetails();
         switch (randomIntBetween(0, 3)) {
@@ -137,7 +137,7 @@ public class EvalQueryQualityTests extends ESTestCase {
             id = id + "_";
             break;
         case 1:
-            qualityLevel = qualityLevel + 0.1;
+            metricScore = metricScore + 0.1;
             break;
         case 2:
             if (metricDetails == null) {
@@ -152,7 +152,7 @@ public class EvalQueryQualityTests extends ESTestCase {
         default:
             throw new IllegalStateException("The test should only allow four parameters mutated");
         }
-        EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel);
+        EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, metricScore);
         evalQueryQuality.setMetricDetails(metricDetails);
         evalQueryQuality.addHitsAndRatings(ratedHits);
         return evalQueryQuality;
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java
index e2be8696e66..fe33c246f7d 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java
@@ -76,10 +76,10 @@ public class ExpectedReciprocalRankTests extends ESTestCase {
         Integer[] relevanceRatings = new Integer[] { 3, 2, 0, 1};
         SearchHit[] hits = createSearchHits(rated, relevanceRatings);
         ExpectedReciprocalRank err = new ExpectedReciprocalRank(3, 0, 3);
-        assertEquals(0.8984375, err.evaluate("id", hits, rated).getQualityLevel(), DELTA);
+        assertEquals(0.8984375, err.evaluate("id", hits, rated).metricScore(), DELTA);
         // take 4th rank into window
         err = new ExpectedReciprocalRank(3, 0, 4);
-        assertEquals(0.8984375 + 0.00244140625, err.evaluate("id", hits, rated).getQualityLevel(), DELTA);
+        assertEquals(0.8984375 + 0.00244140625, err.evaluate("id", hits, rated).metricScore(), DELTA);
     }
 
     /**
@@ -102,11 +102,11 @@ public class ExpectedReciprocalRankTests extends ESTestCase {
         SearchHit[] hits = createSearchHits(rated, relevanceRatings);
         ExpectedReciprocalRank err = new ExpectedReciprocalRank(3, null, 4);
         EvalQueryQuality evaluation = err.evaluate("id", hits, rated);
-        assertEquals(0.875 + 0.00390625, evaluation.getQualityLevel(), DELTA);
+        assertEquals(0.875 + 0.00390625, evaluation.metricScore(), DELTA);
         assertEquals(1, ((ExpectedReciprocalRank.Detail) evaluation.getMetricDetails()).getUnratedDocs());
         // if we supply e.g. 2 as unknown docs rating, it should be the same as in the other test above
         err = new ExpectedReciprocalRank(3, 2, 4);
-        assertEquals(0.8984375 + 0.00244140625, err.evaluate("id", hits, rated).getQualityLevel(), DELTA);
+        assertEquals(0.8984375 + 0.00244140625, err.evaluate("id", hits, rated).metricScore(), DELTA);
     }
 
     private SearchHit[] createSearchHits(List<RatedDocument> rated, Integer[] relevanceRatings) {
@@ -126,7 +126,7 @@ public class ExpectedReciprocalRankTests extends ESTestCase {
      */
     public void testNoResults() throws Exception {
         ExpectedReciprocalRank err = new ExpectedReciprocalRank(5, 0, 10);
-        assertEquals(0.0, err.evaluate("id", new SearchHit[0], Collections.emptyList()).getQualityLevel(), DELTA);
+        assertEquals(0.0, err.evaluate("id", new SearchHit[0], Collections.emptyList()).metricScore(), DELTA);
     }
 
     public void testParseFromXContent() throws IOException {
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
index f88b0cc6634..fdb64806d5c 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
@@ -95,14 +95,14 @@ public class MeanReciprocalRankTests extends ESTestCase {
 
         int rankAtFirstRelevant = relevantAt + 1;
         EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
-        assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE);
+        assertEquals(1.0 / rankAtFirstRelevant, evaluation.metricScore(), Double.MIN_VALUE);
         assertEquals(rankAtFirstRelevant, ((MeanReciprocalRank.Detail) evaluation.getMetricDetails()).getFirstRelevantRank());
 
         // check that if we have fewer search hits than relevant doc position,
-        // we don't find any result and get 0.0 quality level
+        // we don't find any result and get 0.0 score
         reciprocalRank = new MeanReciprocalRank();
         evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), ratedDocs);
-        assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
+        assertEquals(0.0, evaluation.metricScore(), Double.MIN_VALUE);
     }
 
     public void testEvaluationOneRelevantInResults() {
@@ -120,7 +120,7 @@ public class MeanReciprocalRankTests extends ESTestCase {
         }
 
         EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
-        assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE);
+        assertEquals(1.0 / (relevantAt + 1), evaluation.metricScore(), Double.MIN_VALUE);
         assertEquals(relevantAt + 1, ((MeanReciprocalRank.Detail) evaluation.getMetricDetails()).getFirstRelevantRank());
     }
 
@@ -140,7 +140,7 @@ public class MeanReciprocalRankTests extends ESTestCase {
 
         MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(2, 10);
         EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated);
-        assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001);
+        assertEquals((double) 1 / 3, evaluation.metricScore(), 0.00001);
         assertEquals(3, ((MeanReciprocalRank.Detail) evaluation.getMetricDetails()).getFirstRelevantRank());
     }
 
@@ -158,13 +158,13 @@ public class MeanReciprocalRankTests extends ESTestCase {
         SearchHit[] hits = createSearchHits(0, 9, "test");
         List<RatedDocument> ratedDocs = new ArrayList<>();
         EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
-        assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
+        assertEquals(0.0, evaluation.metricScore(), Double.MIN_VALUE);
     }
 
     public void testNoResults() throws Exception {
         SearchHit[] hits = new SearchHit[0];
         EvalQueryQuality evaluated = (new MeanReciprocalRank()).evaluate("id", hits, Collections.emptyList());
-        assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
+        assertEquals(0.0d, evaluated.metricScore(), 0.00001);
         assertEquals(-1, ((MeanReciprocalRank.Detail) evaluated.getMetricDetails()).getFirstRelevantRank());
     }
 
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java
index c0035d5dbb7..73149d5a8aa 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java
@@ -53,7 +53,7 @@ public class PrecisionAtKTests extends ESTestCase {
         List<RatedDocument> rated = new ArrayList<>();
         rated.add(createRatedDoc("test", "0", RELEVANT_RATING_1));
         EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated, "test"), rated);
-        assertEquals(1, evaluated.getQualityLevel(), 0.00001);
+        assertEquals(1, evaluated.metricScore(), 0.00001);
         assertEquals(1, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(1, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
     }
@@ -66,7 +66,7 @@ public class PrecisionAtKTests extends ESTestCase {
         rated.add(createRatedDoc("test", "3", RELEVANT_RATING_1));
         rated.add(createRatedDoc("test", "4", IRRELEVANT_RATING_0));
         EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated, "test"), rated);
-        assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001);
+        assertEquals((double) 4 / 5, evaluated.metricScore(), 0.00001);
         assertEquals(4, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(5, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
     }
@@ -85,7 +85,7 @@ public class PrecisionAtKTests extends ESTestCase {
         rated.add(createRatedDoc("test", "4", 4));
         PrecisionAtK precisionAtN = new PrecisionAtK(2, false, 5);
         EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test"), rated);
-        assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001);
+        assertEquals((double) 3 / 5, evaluated.metricScore(), 0.00001);
         assertEquals(3, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(5, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
     }
@@ -99,7 +99,7 @@ public class PrecisionAtKTests extends ESTestCase {
         rated.add(createRatedDoc("test", "2", IRRELEVANT_RATING_0));
         // the following search hits contain only the last three documents
         EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated.subList(2, 5), "test"), rated);
-        assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001);
+        assertEquals((double) 2 / 3, evaluated.metricScore(), 0.00001);
         assertEquals(2, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(3, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
     }
@@ -114,14 +114,14 @@ public class PrecisionAtKTests extends ESTestCase {
         searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
 
         EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", searchHits, rated);
-        assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001);
+        assertEquals((double) 2 / 3, evaluated.metricScore(), 0.00001);
         assertEquals(2, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(3, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
 
         // also try with setting `ignore_unlabeled`
         PrecisionAtK prec = new PrecisionAtK(1, true, 10);
         evaluated = prec.evaluate("id", searchHits, rated);
-        assertEquals((double) 2 / 2, evaluated.getQualityLevel(), 0.00001);
+        assertEquals((double) 2 / 2, evaluated.metricScore(), 0.00001);
         assertEquals(2, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(2, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
     }
@@ -133,14 +133,14 @@ public class PrecisionAtKTests extends ESTestCase {
             hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
         }
         EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList());
-        assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
+        assertEquals(0.0d, evaluated.metricScore(), 0.00001);
         assertEquals(0, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(5, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
 
         // also try with setting `ignore_unlabeled`
         PrecisionAtK prec = new PrecisionAtK(1, true, 10);
         evaluated = prec.evaluate("id", hits, Collections.emptyList());
-        assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
+        assertEquals(0.0d, evaluated.metricScore(), 0.00001);
         assertEquals(0, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(0, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
     }
@@ -148,7 +148,7 @@ public class PrecisionAtKTests extends ESTestCase {
     public void testNoResults() throws Exception {
         SearchHit[] hits = new SearchHit[0];
         EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList());
-        assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
+        assertEquals(0.0d, evaluated.metricScore(), 0.00001);
         assertEquals(0, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRelevantRetrieved());
         assertEquals(0, ((PrecisionAtK.Detail) evaluated.getMetricDetails()).getRetrieved());
     }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
index 28200e7d5a0..7d594c852da 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
@@ -114,7 +114,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         // the expected Prec@ for the first query is 4/6 and the expected Prec@ for the
         // second is 1/6, divided by 2 to get the average
         double expectedPrecision = (1.0 / 6.0 + 4.0 / 6.0) / 2.0;
-        assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
+        assertEquals(expectedPrecision, response.getMetricScore(), Double.MIN_VALUE);
         Set<Entry<String, EvalQueryQuality>> entrySet = response.getPartialResults().entrySet();
         assertEquals(2, entrySet.size());
         for (Entry<String, EvalQueryQuality> entry : entrySet) {
@@ -157,7 +157,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         // if we look only at top 3 documente, the expected P@3 for the first query is
         // 2/3 and the expected Prec@ for the second is 1/3, divided by 2 to get the average
         expectedPrecision = (1.0 / 3.0 + 2.0 / 3.0) / 2.0;
-        assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
+        assertEquals(expectedPrecision, response.getMetricScore(), Double.MIN_VALUE);
     }
 
     /**
@@ -186,7 +186,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
                 new RankEvalRequest(task, new String[] { TEST_INDEX }));
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
-        assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getEvaluationResult(), 10E-14);
+        assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getMetricScore(), 10E-14);
 
         // test that a different window size k affects the result
         metric = new DiscountedCumulativeGain(false, null, 3);
@@ -195,7 +195,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));
 
         response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
-        assertEquals(12.39278926071437, response.getEvaluationResult(), 10E-14);
+        assertEquals(12.39278926071437, response.getMetricScore(), 10E-14);
     }
 
     public void testMRRRequest() {
@@ -218,7 +218,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         // the expected reciprocal rank for the berlin_query is 1/1
         // dividing by 2 to get the average
         double expectedMRR = (1.0 + 1.0 / 5.0) / 2.0;
-        assertEquals(expectedMRR, response.getEvaluationResult(), 0.0);
+        assertEquals(expectedMRR, response.getMetricScore(), 0.0);
 
         // test that a different window size k affects the result
         metric = new MeanReciprocalRank(1, 3);
@@ -231,7 +231,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         // the reciprocal rank for the berlin_query is 1/1
         // dividing by 2 to get the average
         expectedMRR = 1.0 / 2.0;
-        assertEquals(expectedMRR, response.getEvaluationResult(), 0.0);
+        assertEquals(expectedMRR, response.getMetricScore(), 0.0);
     }
 
     /**
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
index 1e94e869d25..673808f8369 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
@@ -102,7 +102,7 @@ public class RankEvalResponseTests extends ESTestCase {
             try (StreamInput in = output.bytes().streamInput()) {
                 RankEvalResponse deserializedResponse = new RankEvalResponse();
                 deserializedResponse.readFrom(in);
-                assertEquals(randomResponse.getEvaluationResult(), deserializedResponse.getEvaluationResult(), Double.MIN_VALUE);
+                assertEquals(randomResponse.getMetricScore(), deserializedResponse.getMetricScore(), Double.MIN_VALUE);
                 assertEquals(randomResponse.getPartialResults(), deserializedResponse.getPartialResults());
                 assertEquals(randomResponse.getFailures().keySet(), deserializedResponse.getFailures().keySet());
                 assertNotSame(randomResponse, deserializedResponse);
@@ -130,7 +130,7 @@ public class RankEvalResponseTests extends ESTestCase {
         assertNotSame(testItem, parsedItem);
         // We cannot check equality of object here because some information (e.g.
         // SearchHit#shard)  cannot fully be parsed back.
-        assertEquals(testItem.getEvaluationResult(), parsedItem.getEvaluationResult(), 0.0);
+        assertEquals(testItem.getMetricScore(), parsedItem.getMetricScore(), 0.0);
         assertEquals(testItem.getPartialResults().keySet(), parsedItem.getPartialResults().keySet());
         for (EvalQueryQuality metricDetail : testItem.getPartialResults().values()) {
             EvalQueryQuality parsedEvalQueryQuality = parsedItem.getPartialResults().get(metricDetail.getId());
@@ -154,10 +154,10 @@ public class RankEvalResponseTests extends ESTestCase {
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
         String xContent = BytesReference.bytes(response.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString();
         assertEquals(("{" +
-                "    \"quality_level\": 0.123," +
+                "    \"metric_score\": 0.123," +
                 "    \"details\": {" +
                 "        \"coffee_query\": {" +
-                "            \"quality_level\": 0.1," +
+                "            \"metric_score\": 0.1," +
                 "            \"unrated_docs\": [{\"_index\":\"index\",\"_id\":\"456\"}]," +
                 "            \"hits\":[{\"hit\":{\"_index\":\"index\",\"_type\":\"\",\"_id\":\"123\",\"_score\":1.0}," +
                 "                       \"rating\":5}," +
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
index 62c246fb320..fe877b37a68 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
@@ -71,8 +71,8 @@ setup:
           "metric" : { "precision": { "ignore_unlabeled" : true }}
         }
 
-  - match: { quality_level: 1}
-  - match: { details.amsterdam_query.quality_level: 1.0}
+  - match: { metric_score: 1}
+  - match: { details.amsterdam_query.metric_score: 1.0}
   - match: { details.amsterdam_query.unrated_docs:  [ {"_index": "foo", "_id": "doc4"}]}
   - match: { details.amsterdam_query.metric_details.precision: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}}
   
@@ -84,7 +84,7 @@ setup:
   - match: { details.amsterdam_query.hits.2.hit._id:  "doc4"}
   - is_false: details.amsterdam_query.hits.2.rating
   
-  - match: { details.berlin_query.quality_level: 1.0}
+  - match: { details.berlin_query.metric_score: 1.0}
   - match: { details.berlin_query.unrated_docs:  [ {"_index": "foo", "_id": "doc4"}]}
   - match: { details.berlin_query.metric_details.precision: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}}
   - length: { details.berlin_query.hits: 2}
@@ -118,9 +118,9 @@ setup:
           "metric" : { "precision": { "ignore_unlabeled" : true }}
         }
 
-  - match: { quality_level: 1}
-  - match: { details.amsterdam_query.quality_level: 1.0}
-  - match: { details.berlin_query.quality_level: 1.0}
+  - match: { metric_score: 1}
+  - match: { details.amsterdam_query.metric_score: 1.0}
+  - match: { details.berlin_query.metric_score: 1.0}
 
 ---
 "Mean Reciprocal Rank":
@@ -150,14 +150,14 @@ setup:
         }
 
 # average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663
-  - gt: {quality_level: 0.416}
-  - lt: {quality_level: 0.417}
-  - gt: {details.amsterdam_query.quality_level: 0.333}
-  - lt: {details.amsterdam_query.quality_level: 0.334}
+  - gt: {metric_score: 0.416}
+  - lt: {metric_score: 0.417}
+  - gt: {details.amsterdam_query.metric_score: 0.333}
+  - lt: {details.amsterdam_query.metric_score: 0.334}
   - match: {details.amsterdam_query.metric_details.mean_reciprocal_rank: {"first_relevant": 3}}
   - match: {details.amsterdam_query.unrated_docs:  [ {"_index": "foo", "_id": "doc2"},
                                                                {"_index": "foo", "_id": "doc3"} ]}
-  - match: {details.berlin_query.quality_level: 0.5}
+  - match: {details.berlin_query.metric_score: 0.5}
   - match: {details.berlin_query.metric_details.mean_reciprocal_rank: {"first_relevant": 2}}
   - match: {details.berlin_query.unrated_docs:  [ {"_index": "foo", "_id": "doc1"}]}
 
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
index baf10f1542c..1b159775d5c 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
@@ -69,10 +69,10 @@
           "metric" : { "dcg": {}}
         }
 
-  - gt: {quality_level: 13.848263 }
-  - lt: {quality_level: 13.848264 }
-  - gt: {details.dcg_query.quality_level: 13.848263}
-  - lt: {details.dcg_query.quality_level: 13.848264}
+  - gt: {metric_score: 13.848263 }
+  - lt: {metric_score: 13.848264 }
+  - gt: {details.dcg_query.metric_score: 13.848263}
+  - lt: {details.dcg_query.metric_score: 13.848264}
   - match: {details.dcg_query.unrated_docs: [ ]}
 
 # reverse the order in which the results are returned (less relevant docs first)
@@ -96,10 +96,10 @@
           "metric" : { "dcg": { }}
         }
 
-  - gt: {quality_level: 10.299674}
-  - lt: {quality_level: 10.299675}
-  - gt: {details.dcg_query_reverse.quality_level: 10.299674}
-  - lt: {details.dcg_query_reverse.quality_level: 10.299675}
+  - gt: {metric_score: 10.299674}
+  - lt: {metric_score: 10.299675}
+  - gt: {details.dcg_query_reverse.metric_score: 10.299674}
+  - lt: {details.dcg_query_reverse.metric_score: 10.299675}
   - match: {details.dcg_query_reverse.unrated_docs: [ ]}
 
 # if we mix both, we should get the average
@@ -134,11 +134,11 @@
           "metric" : { "dcg": { }}
         }
 
-  - gt: {quality_level: 12.073969}
-  - lt: {quality_level: 12.073970}
-  - gt: {details.dcg_query.quality_level: 13.848263}
-  - lt: {details.dcg_query.quality_level: 13.848264}
+  - gt: {metric_score: 12.073969}
+  - lt: {metric_score: 12.073970}
+  - gt: {details.dcg_query.metric_score: 13.848263}
+  - lt: {details.dcg_query.metric_score: 13.848264}
   - match: {details.dcg_query.unrated_docs: [ ]}
-  - gt: {details.dcg_query_reverse.quality_level: 10.299674}
-  - lt: {details.dcg_query_reverse.quality_level: 10.299675}
+  - gt: {details.dcg_query_reverse.metric_score: 10.299674}
+  - lt: {details.dcg_query_reverse.metric_score: 10.299675}
   - match: {details.dcg_query_reverse.unrated_docs: [ ]}
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
index d6119ad3a9e..42627a2590e 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
@@ -34,8 +34,8 @@
           "metric" : { "precision": { "ignore_unlabeled" : true }}
         }
 
-  - match: { quality_level: 1}
-  - match: { details.amsterdam_query.quality_level: 1.0}
+  - match: { metric_score: 1}
+  - match: { details.amsterdam_query.metric_score: 1.0}
   - match: { details.amsterdam_query.unrated_docs:  [ ]}
   - match: { details.amsterdam_query.metric_details.precision: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}}
 
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml
index 5e0082d213c..fef25c3fc41 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml
@@ -84,7 +84,7 @@ setup:
           "metric" : { "precision": { }}
         }
 
-  - match: {quality_level: 0.9}
+  - match: {metric_score: 0.9}
   - match: {details.amsterdam_query.unrated_docs.0._id:  "6"}
 
 ---
diff --git a/x-pack/qa/core-rest-tests-with-security/src/test/resources/rest-api-spec/test/rankeval/10_rankeval.yml b/x-pack/qa/core-rest-tests-with-security/src/test/resources/rest-api-spec/test/rankeval/10_rankeval.yml
index 6dae2bb2a67..47203779093 100644
--- a/x-pack/qa/core-rest-tests-with-security/src/test/resources/rest-api-spec/test/rankeval/10_rankeval.yml
+++ b/x-pack/qa/core-rest-tests-with-security/src/test/resources/rest-api-spec/test/rankeval/10_rankeval.yml
@@ -52,5 +52,5 @@
           "metric" : { "precision": { "ignore_unlabeled" : true }}
         }
 
-  - match: { quality_level: 1 }
+  - match: { metric_score: 1 }
 

From 55a2d3e0dd6e3f1ac6d37790c8ee5342a0fec764 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 Jul 2018 16:50:15 -0400
Subject: [PATCH 145/260] Switch monitoring to new style Requests (#32255)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `x-pack/plugin/monitoring` project to use the new
versions.
---
 .../http/PublishableHttpResource.java         |  33 +++-
 .../exporter/http/VersionHttpResource.java    |  11 +-
 ...stractPublishableHttpResourceTestCase.java |  43 ++++-
 .../http/HttpExporterResourceTests.java       | 171 +++++++++---------
 .../http/PublishableHttpResourceTests.java    |  63 +++++--
 .../http/VersionHttpResourceTests.java        |  14 +-
 6 files changed, 198 insertions(+), 137 deletions(-)

diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java
index 168ccff44c4..a804703a530 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java
@@ -9,6 +9,7 @@ import org.apache.http.HttpEntity;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -311,13 +312,15 @@ public abstract class PublishableHttpResource extends HttpResource {
                                                               final Set<Integer> exists, final Set<Integer> doesNotExist) {
         logger.trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, resourceOwnerName, resourceOwnerType);
 
-        final Set<Integer> expectedResponseCodes = Sets.union(exists, doesNotExist);
+
+        final Request request = new Request("GET", resourceBasePath + "/" + resourceName);
+        addParameters(request);
         // avoid exists and DNE parameters from being an exception by default
-        final Map<String, String> getParameters = new HashMap<>(parameters);
-        getParameters.put("ignore", expectedResponseCodes.stream().map(i -> i.toString()).collect(Collectors.joining(",")));
+        final Set<Integer> expectedResponseCodes = Sets.union(exists, doesNotExist);
+        request.addParameter("ignore", expectedResponseCodes.stream().map(i -> i.toString()).collect(Collectors.joining(",")));
 
         try {
-            final Response response = client.performRequest("GET", resourceBasePath + "/" + resourceName, getParameters);
+            final Response response = client.performRequest(request);
             final int statusCode = response.getStatusLine().getStatusCode();
 
             // checking the content is the job of whoever called this function by checking the tuple's response
@@ -385,8 +388,12 @@ public abstract class PublishableHttpResource extends HttpResource {
 
         boolean success = false;
 
+        final Request request = new Request("PUT", resourceBasePath + "/" + resourceName);
+        addParameters(request);
+        request.setEntity(body.get());
+
         try {
-            final Response response = client.performRequest("PUT", resourceBasePath + "/" + resourceName, parameters, body.get());
+            final Response response = client.performRequest(request);
             final int statusCode = response.getStatusLine().getStatusCode();
 
             // 200 or 201
@@ -431,12 +438,15 @@ public abstract class PublishableHttpResource extends HttpResource {
 
         boolean success = false;
 
-        // avoid 404 being an exception by default
-        final Map<String, String> deleteParameters = new HashMap<>(parameters);
-        deleteParameters.putIfAbsent("ignore", Integer.toString(RestStatus.NOT_FOUND.getStatus()));
+        Request request = new Request("DELETE", resourceBasePath + "/" + resourceName);
+        addParameters(request);
+        if (false == parameters.containsKey("ignore")) {
+            // avoid 404 being an exception by default
+            request.addParameter("ignore", Integer.toString(RestStatus.NOT_FOUND.getStatus()));
+        }
 
         try {
-            final Response response = client.performRequest("DELETE", resourceBasePath + "/" + resourceName, deleteParameters);
+            final Response response = client.performRequest(request);
             final int statusCode = response.getStatusLine().getStatusCode();
 
             // 200 or 404 (not found is just as good as deleting it!)
@@ -498,4 +508,9 @@ public abstract class PublishableHttpResource extends HttpResource {
         return true;
     }
 
+    private void addParameters(Request request) {
+        for (Map.Entry<String, String> param : parameters.entrySet()) {
+            request.addParameter(param.getKey(), param.getValue());
+        }
+    }
 }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResource.java
index bbc86d37774..eec9162e7ed 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResource.java
@@ -9,6 +9,7 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.logging.Loggers;
@@ -16,7 +17,6 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
 
@@ -27,11 +27,6 @@ public class VersionHttpResource extends HttpResource {
 
     private static final Logger logger = Loggers.getLogger(VersionHttpResource.class);
 
-    /**
-     * The parameters to pass with every version request to limit the output to just the version number.
-     */
-    public static final Map<String, String> PARAMETERS = Collections.singletonMap("filter_path", "version.number");
-
     /**
      * The minimum supported version of Elasticsearch.
      */
@@ -59,7 +54,9 @@ public class VersionHttpResource extends HttpResource {
         logger.trace("checking [{}] to ensure that it supports the minimum version [{}]", resourceOwnerName, minimumVersion);
 
         try {
-            return validateVersion(client.performRequest("GET", "/", PARAMETERS));
+            Request request = new Request("GET", "/");
+            request.addParameter("filter_path", "version.number");
+            return validateVersion(client.performRequest(request));
         } catch (IOException | RuntimeException e) {
             logger.error(
                     (Supplier<?>)() ->
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java
index 208f878543f..ef365042bd5 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java
@@ -11,6 +11,7 @@ import org.apache.http.RequestLine;
 import org.apache.http.StatusLine;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -20,6 +21,8 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
 
 import java.io.IOException;
 import java.util.Map;
@@ -30,10 +33,10 @@ import java.util.stream.Collectors;
 import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.GET_DOES_NOT_EXIST;
 import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.GET_EXISTS;
 import static org.hamcrest.Matchers.is;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
 
 /**
  * Base test helper for any {@link PublishableHttpResource}.
@@ -87,7 +90,9 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
         final ResponseException responseException = responseException("GET", endpoint, failedCheckStatus());
         final Exception e = randomFrom(new IOException("expected"), new RuntimeException("expected"), responseException);
 
-        when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenThrow(e);
+        Request request = new Request("GET", endpoint);
+        addParameters(request, getParameters(resource.getParameters()));
+        when(client.performRequest(request)).thenThrow(e);
 
         assertThat(resource.doCheck(client), is(CheckResponse.ERROR));
     }
@@ -123,7 +128,9 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
         final ResponseException responseException = responseException("DELETE", endpoint, failedCheckStatus());
         final Exception e = randomFrom(new IOException("expected"), new RuntimeException("expected"), responseException);
 
-        when(client.performRequest("DELETE", endpoint, deleteParameters(resource.getParameters()))).thenThrow(e);
+        Request request = new Request("DELETE", endpoint);
+        addParameters(request, deleteParameters(resource.getParameters()));
+        when(client.performRequest(request)).thenThrow(e);
 
         assertThat(resource.doCheck(client), is(CheckResponse.ERROR));
     }
@@ -173,9 +180,15 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
         final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
         final Exception e = randomFrom(new IOException("expected"), new RuntimeException("expected"));
 
-        when(client.performRequest(eq("PUT"), eq(endpoint), eq(resource.getParameters()), any(bodyType))).thenThrow(e);
+        when(client.performRequest(Mockito.any(Request.class))).thenThrow(e);
 
         assertThat(resource.doPublish(client), is(false));
+        ArgumentCaptor<Request> request = ArgumentCaptor.forClass(Request.class);
+        verify(client).performRequest(request.capture());
+        assertThat(request.getValue().getMethod(), is("PUT"));
+        assertThat(request.getValue().getEndpoint(), is(endpoint));
+        assertThat(request.getValue().getParameters(), is(resource.getParameters()));
+        assertThat(request.getValue().getEntity(), instanceOf(bodyType));
     }
 
     protected void assertParameters(final PublishableHttpResource resource) {
@@ -244,7 +257,9 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
                                          final String endpoint, final CheckResponse expected,
                                          final Response response)
             throws IOException {
-        when(client.performRequest("GET", endpoint, expectedParameters)).thenReturn(response);
+        Request request = new Request("GET", endpoint);
+        addParameters(request, expectedParameters);
+        when(client.performRequest(request)).thenReturn(response);
 
         assertThat(resource.doCheck(client), is(expected));
     }
@@ -257,9 +272,14 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
         final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
         final Response response = response("GET", endpoint, status);
 
-        when(client.performRequest(eq("PUT"), eq(endpoint), eq(resource.getParameters()), any(bodyType))).thenReturn(response);
+        ArgumentCaptor<Request> request = ArgumentCaptor.forClass(Request.class);
+        when(client.performRequest(request.capture())).thenReturn(response);
 
         assertThat(resource.doPublish(client), is(expected));
+        assertThat(request.getValue().getMethod(), is("PUT"));
+        assertThat(request.getValue().getEndpoint(), is(endpoint));
+        assertThat(request.getValue().getParameters(), is(resource.getParameters()));
+        assertThat(request.getValue().getEntity(), instanceOf(bodyType));
     }
 
     protected void doCheckAsDeleteWithStatusCode(final PublishableHttpResource resource,
@@ -277,7 +297,9 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
                                                  final String endpoint, final CheckResponse expected,
                                                  final Response response)
             throws IOException {
-        when(client.performRequest("DELETE", endpoint, deleteParameters(resource.getParameters()))).thenReturn(response);
+        Request request = new Request("DELETE", endpoint);
+        addParameters(request, deleteParameters(resource.getParameters()));
+        when(client.performRequest(request)).thenReturn(response);
 
         assertThat(resource.doCheck(client), is(expected));
     }
@@ -427,4 +449,9 @@ public abstract class AbstractPublishableHttpResourceTestCase extends ESTestCase
         return entity;
     }
 
+    protected void addParameters(Request request, Map<String, String> parameters) {
+        for (Map.Entry<String, String> param : parameters.entrySet()) {
+            request.addParameter(param.getKey(), param.getValue());
+        }
+    }
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
index 355a5644bdd..fdf67602af6 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
@@ -10,6 +10,7 @@ import org.apache.http.StatusLine;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -23,6 +24,9 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.TypeSafeMatcher;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -37,10 +41,9 @@ import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplat
 import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.DOES_NOT_EXIST;
 import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.CheckResponse.EXISTS;
 import static org.hamcrest.Matchers.hasSize;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyMapOf;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Matchers.startsWith;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
+import static org.mockito.Matchers.argThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -101,7 +104,8 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
         final HttpEntity entity = new StringEntity("{\"version\":{\"number\":\"unknown\"}}", ContentType.APPLICATION_JSON);
 
         when(versionResponse.getEntity()).thenReturn(entity);
-        when(client.performRequest(eq("GET"), eq("/"), anyMapOf(String.class, String.class))).thenReturn(versionResponse);
+        when(client.performRequest(argThat(new RequestMatcher(is("GET"), is("/")))))
+            .thenReturn(versionResponse);
 
         assertTrue(resources.isDirty());
         assertFalse(resources.checkAndPublish(client));
@@ -140,15 +144,15 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             final List<Response> otherResponses = getTemplateResponses(1, successful, unsuccessful);
 
             // last check fails implies that N - 2 publishes succeeded!
-            when(client.performRequest(eq("GET"), startsWith("/_template/"), anyMapOf(String.class, String.class)))
-                    .thenReturn(first, otherResponses.toArray(new Response[otherResponses.size()]))
-                    .thenThrow(exception);
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_template/")))))
+                .thenReturn(first, otherResponses.toArray(new Response[otherResponses.size()]))
+                .thenThrow(exception);
             whenSuccessfulPutTemplates(otherResponses.size() + 1);
 
             expectedGets += 1 + successful + unsuccessful;
             expectedPuts = (successfulFirst ? 0 : 1) + unsuccessful;
         } else {
-            when(client.performRequest(eq("GET"), startsWith("/_template/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_template/")))))
                     .thenThrow(exception);
         }
 
@@ -185,7 +189,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             whenGetTemplates(successful, unsuccessful + 2);
 
             // previous publishes must have succeeded
-            when(client.performRequest(eq("PUT"), startsWith("/_template/"), anyMapOf(String.class, String.class), any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_template/")))))
                     .thenReturn(firstSuccess, otherResponses.toArray(new Response[otherResponses.size()]))
                     .thenThrow(exception);
 
@@ -197,7 +201,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             // fail the check so that it has to attempt the PUT
             whenGetTemplates(0, 1);
 
-            when(client.performRequest(eq("PUT"), startsWith("/_template/"), anyMapOf(String.class, String.class), any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_template/")))))
                     .thenThrow(exception);
         }
 
@@ -238,7 +242,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             }
 
             // last check fails
-            when(client.performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_ingest/pipeline/")))))
                     .thenReturn(first)
                     .thenThrow(exception);
             if (successfulFirst == false) {
@@ -248,7 +252,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             expectedGets = EXPECTED_PIPELINES;
             expectedPuts = successfulFirst ? 0 : 1;
         } else {
-            when(client.performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_ingest/pipeline/")))))
                     .thenThrow(exception);
         }
 
@@ -285,10 +289,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             whenGetPipelines(0, 2);
 
             // previous publishes must have succeeded
-            when(client.performRequest(eq("PUT"),
-                                       startsWith("/_ingest/pipeline/"),
-                                       anyMapOf(String.class, String.class),
-                                       any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_ingest/pipeline/")))))
                     .thenReturn(firstSuccess)
                     .thenThrow(exception);
 
@@ -300,10 +301,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             // fail the check so that it has to attempt the PUT
             whenGetPipelines(0, 1);
 
-            when(client.performRequest(eq("PUT"),
-                                       startsWith("/_ingest/pipeline/"),
-                                       anyMapOf(String.class, String.class),
-                                       any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_ingest/pipeline/")))))
                     .thenThrow(exception);
         }
 
@@ -334,7 +332,8 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
         whenSuccessfulPutPipelines(unsuccessfulGetPipelines);
 
         // there's only one check
-        when(client.performRequest(eq("GET"), eq("/_xpack"), anyMapOf(String.class, String.class))).thenThrow(exception);
+        when(client.performRequest(argThat(new RequestMatcher(is("GET"), is("/_xpack")))))
+                .thenThrow(exception);
 
         assertTrue(resources.isDirty());
         assertFalse(resources.checkAndPublish(client));
@@ -382,7 +381,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
                 final List<Response> otherResponses = getWatcherResponses(1, successful, unsuccessful);
 
                 // last check fails implies that N - 2 publishes succeeded!
-                when(client.performRequest(eq("GET"), startsWith("/_xpack/watcher/watch/"), anyMapOf(String.class, String.class)))
+                when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_xpack/watcher/watch/")))))
                         .thenReturn(first, otherResponses.toArray(new Response[otherResponses.size()]))
                         .thenThrow(exception);
                 whenSuccessfulPutWatches(otherResponses.size() + 1);
@@ -398,7 +397,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
                 // there is no form of an unsuccessful delete; only success or error
                 final List<Response> responses = successfulDeleteResponses(successful);
 
-                when(client.performRequest(eq("DELETE"), startsWith("/_xpack/watcher/watch/"), anyMapOf(String.class, String.class)))
+                when(client.performRequest(argThat(new RequestMatcher(is("DELETE"), startsWith("/_xpack/watcher/watch/")))))
                         .thenReturn(responses.get(0), responses.subList(1, successful).toArray(new Response[successful - 1]))
                         .thenThrow(exception);
 
@@ -407,7 +406,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
         } else {
             final String method = validLicense ? "GET" : "DELETE";
 
-            when(client.performRequest(eq(method), startsWith("/_xpack/watcher/watch/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is(method), startsWith("/_xpack/watcher/watch/")))))
                     .thenThrow(exception);
         }
 
@@ -463,10 +462,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             whenGetWatches(successful, unsuccessful + 2);
 
             // previous publishes must have succeeded
-            when(client.performRequest(eq("PUT"),
-                                       startsWith("/_xpack/watcher/watch/"),
-                                       anyMapOf(String.class, String.class),
-                                       any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_xpack/watcher/watch/")))))
                     .thenReturn(firstSuccess, otherResponses.toArray(new Response[otherResponses.size()]))
                     .thenThrow(exception);
 
@@ -478,10 +474,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
             // fail the check so that it has to attempt the PUT
             whenGetWatches(0, 1);
 
-            when(client.performRequest(eq("PUT"),
-                                       startsWith("/_xpack/watcher/watch/"),
-                                       anyMapOf(String.class, String.class),
-                                       any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_xpack/watcher/watch/")))))
                     .thenThrow(exception);
         }
 
@@ -715,17 +708,18 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
         final HttpEntity entity = new StringEntity("{\"version\":{\"number\":\"" + Version.CURRENT + "\"}}", ContentType.APPLICATION_JSON);
 
         when(versionResponse.getEntity()).thenReturn(entity);
-        when(client.performRequest(eq("GET"), eq("/"), anyMapOf(String.class, String.class))).thenReturn(versionResponse);
+        when(client.performRequest(argThat(new RequestMatcher(is("GET"), is("/")))))
+                .thenReturn(versionResponse);
     }
 
     private void whenGetTemplates(final int successful, final int unsuccessful) throws IOException {
         final List<Response> gets = getTemplateResponses(0, successful, unsuccessful);
 
         if (gets.size() == 1) {
-            when(client.performRequest(eq("GET"), startsWith("/_template/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_template/")))))
                     .thenReturn(gets.get(0));
         } else {
-            when(client.performRequest(eq("GET"), startsWith("/_template/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_template/")))))
                     .thenReturn(gets.get(0), gets.subList(1, gets.size()).toArray(new Response[gets.size() - 1]));
         }
     }
@@ -735,10 +729,10 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
 
         // empty is possible if they all exist
         if (successful == 1) {
-            when(client.performRequest(eq("PUT"), startsWith("/_template/"), anyMapOf(String.class, String.class), any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_template/")))))
                     .thenReturn(successfulPuts.get(0));
         } else if (successful > 1) {
-            when(client.performRequest(eq("PUT"), startsWith("/_template/"), anyMapOf(String.class, String.class), any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_template/")))))
                     .thenReturn(successfulPuts.get(0), successfulPuts.subList(1, successful).toArray(new Response[successful - 1]));
         }
     }
@@ -747,10 +741,10 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
         final List<Response> gets = getPipelineResponses(0, successful, unsuccessful);
 
         if (gets.size() == 1) {
-            when(client.performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_ingest/pipeline/")))))
                     .thenReturn(gets.get(0));
         } else {
-            when(client.performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_ingest/pipeline/")))))
                     .thenReturn(gets.get(0), gets.subList(1, gets.size()).toArray(new Response[gets.size() - 1]));
         }
     }
@@ -760,16 +754,10 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
 
         // empty is possible if they all exist
         if (successful == 1) {
-            when(client.performRequest(eq("PUT"),
-                                       startsWith("/_ingest/pipeline/"),
-                                       anyMapOf(String.class, String.class),
-                                       any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_ingest/pipeline/")))))
                     .thenReturn(successfulPuts.get(0));
         } else if (successful > 1) {
-            when(client.performRequest(eq("PUT"),
-                                       startsWith("/_ingest/pipeline/"),
-                                       anyMapOf(String.class, String.class),
-                                       any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_ingest/pipeline/")))))
                     .thenReturn(successfulPuts.get(0), successfulPuts.subList(1, successful).toArray(new Response[successful - 1]));
         }
     }
@@ -787,7 +775,8 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
         final Response successfulGet = response("GET", "_xpack", successfulCheckStatus(), entity);
 
         // empty is possible if they all exist
-        when(client.performRequest(eq("GET"), eq("/_xpack"), anyMapOf(String.class, String.class))).thenReturn(successfulGet);
+        when(client.performRequest(argThat(new RequestMatcher(is("GET"), is("/_xpack")))))
+                .thenReturn(successfulGet);
     }
 
     private void whenWatcherCannotBeUsed() throws IOException {
@@ -805,17 +794,18 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
         }
 
         // empty is possible if they all exist
-        when(client.performRequest(eq("GET"), eq("/_xpack"), anyMapOf(String.class, String.class))).thenReturn(response);
+        when(client.performRequest(argThat(new RequestMatcher(is("GET"), is("/_xpack")))))
+                .thenReturn(response);
     }
 
     private void whenGetWatches(final int successful, final int unsuccessful) throws IOException {
         final List<Response> gets = getWatcherResponses(0, successful, unsuccessful);
 
         if (gets.size() == 1) {
-            when(client.performRequest(eq("GET"), startsWith("/_xpack/watcher/watch/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_xpack/watcher/watch/")))))
                     .thenReturn(gets.get(0));
         } else {
-            when(client.performRequest(eq("GET"), startsWith("/_xpack/watcher/watch/"), anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_xpack/watcher/watch/")))))
                     .thenReturn(gets.get(0), gets.subList(1, gets.size()).toArray(new Response[gets.size() - 1]));
         }
     }
@@ -825,16 +815,10 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
 
         // empty is possible if they all exist
         if (successful == 1) {
-            when(client.performRequest(eq("PUT"),
-                    startsWith("/_xpack/watcher/watch/"),
-                    anyMapOf(String.class, String.class),
-                    any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_xpack/watcher/watch/")))))
                     .thenReturn(successfulPuts.get(0));
         } else if (successful > 1) {
-            when(client.performRequest(eq("PUT"),
-                    startsWith("/_xpack/watcher/watch/"),
-                    anyMapOf(String.class, String.class),
-                    any(HttpEntity.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_xpack/watcher/watch/")))))
                     .thenReturn(successfulPuts.get(0), successfulPuts.subList(1, successful).toArray(new Response[successful - 1]));
         }
     }
@@ -844,64 +828,55 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
 
         // empty is possible if they all exist
         if (successful == 1) {
-            when(client.performRequest(eq("DELETE"),
-                                       startsWith("/_xpack/watcher/watch/"),
-                                       anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("DELETE"), startsWith("/_xpack/watcher/watch/")))))
                     .thenReturn(successfulDeletes.get(0));
         } else if (successful > 1) {
-            when(client.performRequest(eq("DELETE"),
-                                       startsWith("/_xpack/watcher/watch/"),
-                                       anyMapOf(String.class, String.class)))
+            when(client.performRequest(argThat(new RequestMatcher(is("DELETE"), startsWith("/_xpack/watcher/watch/")))))
                     .thenReturn(successfulDeletes.get(0), successfulDeletes.subList(1, successful).toArray(new Response[successful - 1]));
         }
     }
+
     private void verifyVersionCheck() throws IOException {
-        verify(client).performRequest(eq("GET"), eq("/"), anyMapOf(String.class, String.class));
+        verify(client).performRequest(argThat(new RequestMatcher(is("GET"), is("/"))));
     }
 
     private void verifyGetTemplates(final int called) throws IOException {
-        verify(client, times(called)).performRequest(eq("GET"), startsWith("/_template/"), anyMapOf(String.class, String.class));
+        verify(client, times(called))
+                .performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_template/"))));
     }
 
     private void verifyPutTemplates(final int called) throws IOException {
-        verify(client, times(called)).performRequest(eq("PUT"),                            // method
-                                                     startsWith("/_template/"),            // endpoint
-                                                     anyMapOf(String.class, String.class), // parameters (e.g., timeout)
-                                                     any(HttpEntity.class));               // raw template
+        verify(client, times(called))
+                .performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_template/"))));
     }
 
     private void verifyGetPipelines(final int called) throws IOException {
-        verify(client, times(called)).performRequest(eq("GET"), startsWith("/_ingest/pipeline/"), anyMapOf(String.class, String.class));
+        verify(client, times(called))
+                .performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_ingest/pipeline/"))));
     }
 
     private void verifyPutPipelines(final int called) throws IOException {
-        verify(client, times(called)).performRequest(eq("PUT"),                       // method
-                                                     startsWith("/_ingest/pipeline/"),     // endpoint
-                                                     anyMapOf(String.class, String.class), // parameters (e.g., timeout)
-                                                     any(HttpEntity.class));               // raw template
+        verify(client, times(called))
+                .performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_ingest/pipeline/"))));
     }
 
     private void verifyWatcherCheck() throws IOException {
-        verify(client).performRequest(eq("GET"), eq("/_xpack"), anyMapOf(String.class, String.class));
+        verify(client).performRequest(argThat(new RequestMatcher(is("GET"), is("/_xpack"))));
     }
 
     private void verifyDeleteWatches(final int called) throws IOException {
-        verify(client, times(called)).performRequest(eq("DELETE"),                    // method
-                                                     startsWith("/_xpack/watcher/watch/"), // endpoint
-                                                     anyMapOf(String.class, String.class));// parameters (e.g., timeout)
+        verify(client, times(called))
+                .performRequest(argThat(new RequestMatcher(is("DELETE"), startsWith("/_xpack/watcher/watch/"))));
     }
 
     private void verifyGetWatches(final int called) throws IOException {
-        verify(client, times(called)).performRequest(eq("GET"),
-                                                     startsWith("/_xpack/watcher/watch/"),
-                                                     anyMapOf(String.class, String.class));
+        verify(client, times(called))
+                .performRequest(argThat(new RequestMatcher(is("GET"), startsWith("/_xpack/watcher/watch/"))));
     }
 
     private void verifyPutWatches(final int called) throws IOException {
-        verify(client, times(called)).performRequest(eq("PUT"),                            // method
-                                                     startsWith("/_xpack/watcher/watch/"),      // endpoint
-                                                     anyMapOf(String.class, String.class), // parameters (e.g., timeout)
-                                                     any(HttpEntity.class));               // raw template
+        verify(client, times(called))
+                .performRequest(argThat(new RequestMatcher(is("PUT"), startsWith("/_xpack/watcher/watch/"))));
     }
 
     private ClusterService mockClusterService(final ClusterState state) {
@@ -922,4 +897,24 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
         return state;
     }
 
+    private static class RequestMatcher extends TypeSafeMatcher<Request> {
+        private final Matcher<String> method;
+        private final Matcher<String> endpoint;
+
+        private RequestMatcher(Matcher<String> method, Matcher<String> endpoint) {
+            this.method = method;
+            this.endpoint = endpoint;
+        }
+
+        @Override
+        protected boolean matchesSafely(Request item) {
+            return method.matches(item.getMethod()) && endpoint.matches(item.getEndpoint());
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description.appendText("method is ").appendDescriptionOf(method);
+            description.appendText(" and endpoint is ").appendDescriptionOf(endpoint);
+        }
+    }
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResourceTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResourceTests.java
index 9c217612263..e32effc85f9 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResourceTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResourceTests.java
@@ -10,6 +10,7 @@ import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -61,13 +62,15 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
         final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
         final RestStatus failedStatus = failedCheckStatus();
         final Response response = response("GET", endpoint, failedStatus);
+        final Request request = new Request("GET", endpoint);
+        addParameters(request, getParameters(resource.getParameters()));
 
-        when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenReturn(response);
+        when(client.performRequest(request)).thenReturn(response);
 
         sometimesAssertSimpleCheckForResource(client, logger, resourceBasePath, resourceName, resourceType, CheckResponse.ERROR, response);
 
         verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
-        verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
+        verify(client).performRequest(request);
         verify(logger).error(any(org.apache.logging.log4j.util.Supplier.class), any(ResponseException.class));
 
         verifyNoMoreInteractions(client, logger);
@@ -95,8 +98,10 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
         final Response response = response("GET", endpoint, failedStatus);
         final XContent xContent = mock(XContent.class);
         final int minimumVersion = randomInt();
+        final Request request = new Request("GET", endpoint);
+        addParameters(request, getParameters(resource.getParameters()));
 
-        when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenReturn(response);
+        when(client.performRequest(request)).thenReturn(response);
 
         assertThat(resource.versionCheckForResource(client, logger,
                                                     resourceBasePath, resourceName, resourceType, owner, ownerType,
@@ -104,7 +109,7 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
                    is(CheckResponse.ERROR));
 
         verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
-        verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
+        verify(client).performRequest(request);
         verify(logger).error(any(org.apache.logging.log4j.util.Supplier.class), any(ResponseException.class));
 
         verifyNoMoreInteractions(client, logger);
@@ -117,8 +122,10 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
         final HttpEntity entity = entityForResource(CheckResponse.ERROR, resourceName, minimumVersion);
         final Response response = response("GET", endpoint, okStatus, entity);
         final XContent xContent = mock(XContent.class);
+        final Request request = new Request("GET", endpoint);
+        addParameters(request, getParameters(resource.getParameters()));
 
-        when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenReturn(response);
+        when(client.performRequest(request)).thenReturn(response);
 
         assertThat(resource.versionCheckForResource(client, logger,
                                                     resourceBasePath, resourceName, resourceType, owner, ownerType,
@@ -127,7 +134,7 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
 
         verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
         verify(logger).debug("{} [{}] found on the [{}] {}", resourceType, resourceName, owner, ownerType);
-        verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
+        verify(client).performRequest(request);
         verify(logger).error(any(org.apache.logging.log4j.util.Supplier.class), any(ResponseException.class));
 
         verifyNoMoreInteractions(client, logger);
@@ -140,12 +147,14 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
         final Exception e = randomFrom(new IOException("expected"), new RuntimeException("expected"), responseException);
         final Response response = e == responseException ? responseException.getResponse() : null;
 
-        when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenThrow(e);
+        Request request = new Request("GET", endpoint);
+        addParameters(request, getParameters(resource.getParameters()));
+        when(client.performRequest(request)).thenThrow(e);
 
         sometimesAssertSimpleCheckForResource(client, logger, resourceBasePath, resourceName, resourceType, CheckResponse.ERROR, response);
 
         verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
-        verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
+        verify(client).performRequest(request);
         verify(logger).error(any(org.apache.logging.log4j.util.Supplier.class), eq(e));
 
         verifyNoMoreInteractions(client, logger);
@@ -162,13 +171,16 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
     public void testPutResourceFalseWithException() throws IOException {
         final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
         final Exception e = randomFrom(new IOException("expected"), new RuntimeException("expected"));
+        final Request request = new Request("PUT", endpoint);
+        addParameters(request, resource.getParameters());
+        request.setEntity(entity);
 
-        when(client.performRequest("PUT", endpoint, resource.getParameters(), entity)).thenThrow(e);
+        when(client.performRequest(request)).thenThrow(e);
 
         assertThat(resource.putResource(client, logger, resourceBasePath, resourceName, body, resourceType, owner, ownerType), is(false));
 
         verify(logger).trace("uploading {} [{}] to the [{}] {}", resourceType, resourceName, owner, ownerType);
-        verify(client).performRequest("PUT", endpoint, resource.getParameters(), entity);
+        verify(client).performRequest(request);
         verify(logger).error(any(org.apache.logging.log4j.util.Supplier.class), eq(e));
 
         verifyNoMoreInteractions(client, logger);
@@ -190,13 +202,15 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
         final ResponseException responseException = responseException("DELETE", endpoint, failedStatus);
         final Exception e = randomFrom(new IOException("expected"), new RuntimeException("expected"), responseException);
         final Map<String, String> deleteParameters = deleteParameters(resource.getParameters());
+        final Request request = new Request("DELETE", endpoint);
+        addParameters(request, deleteParameters);
 
-        when(client.performRequest("DELETE", endpoint, deleteParameters)).thenThrow(e);
+        when(client.performRequest(request)).thenThrow(e);
 
         assertThat(resource.deleteResource(client, logger, resourceBasePath, resourceName, resourceType, owner, ownerType), is(false));
 
         verify(logger).trace("deleting {} [{}] from the [{}] {}", resourceType, resourceName, owner, ownerType);
-        verify(client).performRequest("DELETE", endpoint, deleteParameters);
+        verify(client).performRequest(request);
         verify(logger).error(any(org.apache.logging.log4j.util.Supplier.class), eq(e));
 
         verifyNoMoreInteractions(client, logger);
@@ -277,13 +291,15 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
             throws IOException {
         final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
         final Response response = response("GET", endpoint, status);
+        final Request request = new Request("GET", endpoint);
+        addParameters(request, getParameters(resource.getParameters()));
 
-        when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenReturn(response);
+        when(client.performRequest(request)).thenReturn(response);
 
         sometimesAssertSimpleCheckForResource(client, logger, resourceBasePath, resourceName, resourceType, expected, response);
 
         verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
-        verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
+        verify(client).performRequest(request);
 
         if (expected == CheckResponse.EXISTS || expected == CheckResponse.DOES_NOT_EXIST) {
             verify(response).getStatusLine();
@@ -310,8 +326,10 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
         final HttpEntity entity = status == RestStatus.OK ? entityForResource(expected, resourceName, minimumVersion) : null;
         final Response response = response("GET", endpoint, status, entity);
         final XContent xContent = XContentType.JSON.xContent();
+        final Request request = new Request("GET", endpoint);
+        addParameters(request, getParameters(resource.getParameters()));
 
-        when(client.performRequest("GET", endpoint, getParameters(resource.getParameters()))).thenReturn(response);
+        when(client.performRequest(request)).thenReturn(response);
 
         assertThat(resource.versionCheckForResource(client, logger,
                                                     resourceBasePath, resourceName, resourceType, owner, ownerType,
@@ -319,7 +337,7 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
                    is(expected));
 
         verify(logger).trace("checking if {} [{}] exists on the [{}] {}", resourceType, resourceName, owner, ownerType);
-        verify(client).performRequest("GET", endpoint, getParameters(resource.getParameters()));
+        verify(client).performRequest(request);
 
         if (shouldReplace || expected == CheckResponse.EXISTS) {
             verify(response).getStatusLine();
@@ -341,13 +359,16 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
     private void assertPutResource(final RestStatus status, final boolean expected) throws IOException {
         final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
         final Response response = response("PUT", endpoint, status);
+        final Request request = new Request("PUT", endpoint);
+        addParameters(request, resource.getParameters());
+        request.setEntity(entity);
 
-        when(client.performRequest("PUT", endpoint, resource.getParameters(), entity)).thenReturn(response);
+        when(client.performRequest(request)).thenReturn(response);
 
         assertThat(resource.putResource(client, logger, resourceBasePath, resourceName, body, resourceType, owner, ownerType),
                    is(expected));
 
-        verify(client).performRequest("PUT", endpoint, resource.getParameters(), entity);
+        verify(client).performRequest(request);
         verify(response).getStatusLine();
 
         verify(logger).trace("uploading {} [{}] to the [{}] {}", resourceType, resourceName, owner, ownerType);
@@ -388,12 +409,14 @@ public class PublishableHttpResourceTests extends AbstractPublishableHttpResourc
         final String endpoint = concatenateEndpoint(resourceBasePath, resourceName);
         final Response response = response("DELETE", endpoint, status);
         final Map<String, String> deleteParameters = deleteParameters(resource.getParameters());
+        final Request request = new Request("DELETE", endpoint);
+        addParameters(request, deleteParameters);
 
-        when(client.performRequest("DELETE", endpoint, deleteParameters)).thenReturn(response);
+        when(client.performRequest(request)).thenReturn(response);
 
         assertThat(resource.deleteResource(client, logger, resourceBasePath, resourceName, resourceType, owner, ownerType), is(expected));
 
-        verify(client).performRequest("DELETE", endpoint, deleteParameters);
+        verify(client).performRequest(request);
         verify(response).getStatusLine();
 
         verify(logger).trace("deleting {} [{}] from the [{}] {}", resourceType, resourceName, owner, ownerType);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResourceTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResourceTests.java
index d6005b57046..27de4b28cee 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResourceTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResourceTests.java
@@ -6,8 +6,9 @@
 package org.elasticsearch.xpack.monitoring.exporter.http;
 
 import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
+import org.apache.http.nio.entity.NStringEntity;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.test.ESTestCase;
@@ -73,8 +74,9 @@ public class VersionHttpResourceTests extends ESTestCase {
     }
 
     public void testDoCheckAndPublishFailedWithIOException() throws IOException {
-        // request fails for some reason
-        when(client.performRequest("GET", "/", VersionHttpResource.PARAMETERS)).thenThrow(new IOException("expected"));
+        Request request = new Request("GET", "/");
+        request.addParameter("filter_path", "version.number");
+        when(client.performRequest(request)).thenThrow(new IOException("expected"));
 
         final VersionHttpResource resource = new VersionHttpResource(owner, Version.CURRENT);
 
@@ -82,12 +84,14 @@ public class VersionHttpResourceTests extends ESTestCase {
     }
 
     private Response responseForJSON(final String json) throws IOException {
-        final StringEntity entity = new StringEntity(json, ContentType.APPLICATION_JSON);
+        final NStringEntity entity = new NStringEntity(json, ContentType.APPLICATION_JSON);
 
         final Response response = mock(Response.class);
         when(response.getEntity()).thenReturn(entity);
 
-        when(client.performRequest("GET", "/", VersionHttpResource.PARAMETERS)).thenReturn(response);
+        Request request = new Request("GET", "/");
+        request.addParameter("filter_path", "version.number");
+        when(client.performRequest(request)).thenReturn(response);
 
         return response;
     }

From 6ba144ae31a546dbbffc58e0082786a48ec233a7 Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Mon, 23 Jul 2018 18:33:15 -0400
Subject: [PATCH 146/260] Add WeightedAvg metric aggregation (#31037)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds a new single-value metrics aggregation that computes the weighted
average of numeric values that are extracted from the aggregated
documents. These values can be extracted from specific numeric
fields in the documents.

When calculating a regular average, each datapoint has an equal "weight"; it
contributes equally to the final value.  In contrast, weighted averages
scale each datapoint differently.  The amount that each datapoint contributes
to the final value is extracted from the document, or provided by a script.

As a formula, a weighted average is the `∑(value * weight) / ∑(weight)`

A regular average can be thought of as a weighted average where every value has
an implicit weight of `1`.

Closes #15731
---
 docs/build.gradle                             |   4 +-
 docs/reference/aggregations/metrics.asciidoc  |   2 +
 .../metrics/weighted-avg-aggregation.asciidoc | 202 +++++++++
 .../stats/MatrixStatsAggregationBuilder.java  |   4 +-
 .../matrix/stats/MatrixStatsAggregator.java   |   6 +-
 .../stats/MatrixStatsAggregatorFactory.java   |   4 +-
 .../matrix/stats/MatrixStatsParser.java       |   4 +-
 ...luesSource.java => ArrayValuesSource.java} |  14 +-
 ... ArrayValuesSourceAggregationBuilder.java} |  33 +-
 ...> ArrayValuesSourceAggregatorFactory.java} |  14 +-
 ...rser.java => ArrayValuesSourceParser.java} |  40 +-
 .../search.aggregation/260_weighted_avg.yml   |  71 +++
 .../elasticsearch/search/SearchModule.java    |   4 +
 .../aggregations/AggregationBuilders.java     |   8 +
 .../weighted_avg/InternalWeightedAvg.java     | 144 ++++++
 .../weighted_avg/ParsedWeightedAvg.java       |  65 +++
 .../metrics/weighted_avg/WeightedAvg.java     |  32 ++
 .../WeightedAvgAggregationBuilder.java        | 128 ++++++
 .../weighted_avg/WeightedAvgAggregator.java   | 158 +++++++
 .../WeightedAvgAggregatorFactory.java         |  64 +++
 .../support/MultiValuesSource.java            |  93 ++++
 .../MultiValuesSourceAggregationBuilder.java  | 268 +++++++++++
 .../MultiValuesSourceAggregatorFactory.java   |  64 +++
 .../support/MultiValuesSourceFieldConfig.java | 186 ++++++++
 .../support/MultiValuesSourceParseHelper.java |  59 +++
 .../aggregations/support/ValueType.java       |   5 +-
 .../support/ValuesSourceParserHelper.java     |  11 +-
 .../support/ValuesSourceType.java             |  30 +-
 .../WeightedAvgAggregatorTests.java           | 428 ++++++++++++++++++
 .../MultiValuesSourceFieldConfigTests.java    |  38 ++
 .../xpack/rollup/RollupRequestTranslator.java |   2 +-
 31 files changed, 2117 insertions(+), 68 deletions(-)
 create mode 100644 docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc
 rename modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/{MultiValuesSource.java => ArrayValuesSource.java} (87%)
 rename modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/{MultiValuesSourceAggregationBuilder.java => ArrayValuesSourceAggregationBuilder.java} (91%)
 rename modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/{MultiValuesSourceAggregatorFactory.java => ArrayValuesSourceAggregatorFactory.java} (78%)
 rename modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/{MultiValuesSourceParser.java => ArrayValuesSourceParser.java} (86%)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/InternalWeightedAvg.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/ParsedWeightedAvg.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvg.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilder.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregator.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregatorFactory.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregatorTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java

diff --git a/docs/build.gradle b/docs/build.gradle
index 829db4381b0..a67c0217490 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -379,9 +379,9 @@ buildRestTests.setups['exams'] = '''
           refresh: true
           body: |
             {"index":{}}
-            {"grade": 100}
+            {"grade": 100, "weight": 2}
             {"index":{}}
-            {"grade": 50}'''
+            {"grade": 50, "weight": 3}'''
 
 buildRestTests.setups['stored_example_script'] = '''
   # Simple script to load a field. Not really a good example, but a simple one.
diff --git a/docs/reference/aggregations/metrics.asciidoc b/docs/reference/aggregations/metrics.asciidoc
index ae6bee2eb7d..96597564dac 100644
--- a/docs/reference/aggregations/metrics.asciidoc
+++ b/docs/reference/aggregations/metrics.asciidoc
@@ -13,6 +13,8 @@ bucket aggregations (some bucket aggregations enable you to sort the returned bu
 
 include::metrics/avg-aggregation.asciidoc[]
 
+include::metrics/weighted-avg-aggregation.asciidoc[]
+
 include::metrics/cardinality-aggregation.asciidoc[]
 
 include::metrics/extendedstats-aggregation.asciidoc[]
diff --git a/docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc b/docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc
new file mode 100644
index 00000000000..252728a6db3
--- /dev/null
+++ b/docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc
@@ -0,0 +1,202 @@
+[[search-aggregations-metrics-weight-avg-aggregation]]
+=== Weighted Avg Aggregation
+
+A `single-value` metrics aggregation that computes the weighted average of numeric values that are extracted from the aggregated documents.
+These values can be extracted either from specific numeric fields in the documents.
+
+When calculating a regular average, each datapoint has an equal "weight" ... it contributes equally to the final value.  Weighted averages,
+on the other hand, weight each datapoint differently.  The amount that each datapoint contributes to the final value is extracted from the
+document, or provided by a script.
+
+As a formula, a weighted average is the `∑(value * weight) / ∑(weight)`
+
+A regular average can be thought of as a weighted average where every value has an implicit weight of `1`.
+
+.`weighted_avg` Parameters
+|===
+|Parameter Name |Description |Required |Default Value
+|`value` | The configuration for the field or script that provides the values |Required |
+|`weight` | The configuration for the field or script that provides the weights |Required |
+|`format` | The numeric response formatter |Optional |
+|`value_type` | A hint about the values for pure scripts or unmapped fields |Optional |
+|===
+
+The `value` and `weight` objects have per-field specific configuration:
+
+.`value` Parameters
+|===
+|Parameter Name |Description |Required |Default Value
+|`field` | The field that values should be extracted from |Required |
+|`missing` | A value to use if the field is missing entirely |Optional |
+|`script` | A script which provides the values for the document.  This is mutually exclusive with `field` |Optional
+|===
+
+.`weight` Parameters
+|===
+|Parameter Name |Description |Required |Default Value
+|`field` | The field that weights should be extracted from |Required |
+|`missing` | A weight to use if the field is missing entirely |Optional |
+|`script` | A script which provides the weights for the document.  This is mutually exclusive with `field` |Optional
+|===
+
+
+==== Examples
+
+If our documents have a `"grade"` field that holds a 0-100 numeric score, and a `"weight"` field which holds an arbitrary numeric weight,
+we can calculate the weighted average using:
+
+[source,js]
+--------------------------------------------------
+POST /exams/_search
+{
+    "size": 0,
+    "aggs" : {
+        "weighted_grade": {
+            "weighted_avg": {
+                "value": {
+                    "field": "grade"
+                },
+                "weight": {
+                    "field": "weight"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:exams]
+
+Which yields a response like:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "weighted_grade": {
+            "value": 70.0
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+
+While multiple values-per-field are allowed, only one weight is allowed.  If the aggregation encounters
+a document that has more than one weight (e.g. the weight field is a multi-valued field) it will throw an exception.
+If you have this situation, you will need to specify a `script` for the weight field, and use the script
+to combine the multiple values into a single value to be used.
+
+This single weight will be applied independently to each value extracted from the `value` field.
+
+This example show how a single document with multiple values will be averaged with a single weight:
+
+[source,js]
+--------------------------------------------------
+POST /exams/_doc?refresh
+{
+    "grade": [1, 2, 3],
+    "weight": 2
+}
+
+POST /exams/_search
+{
+    "size": 0,
+    "aggs" : {
+        "weighted_grade": {
+            "weighted_avg": {
+                "value": {
+                    "field": "grade"
+                },
+                "weight": {
+                    "field": "weight"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+The three values (`1`, `2`, and `3`) will be included as independent values, all with the weight of `2`:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "weighted_grade": {
+            "value": 2.0
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+The aggregation returns `2.0` as the result, which matches what we would expect when calculating by hand:
+`((1*2) + (2*2) + (3*2)) / (2+2+2) == 2`
+
+==== Script
+
+Both the value and the weight can be derived from a script, instead of a field.  As a simple example, the following
+will add one to the grade and weight in the document using a script:
+
+[source,js]
+--------------------------------------------------
+POST /exams/_search
+{
+    "size": 0,
+    "aggs" : {
+        "weighted_grade": {
+            "weighted_avg": {
+                "value": {
+                    "script": "doc.grade.value + 1"
+                },
+                "weight": {
+                    "script": "doc.weight.value + 1"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:exams]
+
+
+==== Missing values
+
+The `missing` parameter defines how documents that are missing a value should be treated.
+The default behavior is different for `value` and `weight`:
+
+By default, if the `value` field is missing the document is ignored and the aggregation moves on to the next document.
+If the `weight` field is missing, it is assumed to have a weight of `1` (like a normal average).
+
+Both of these defaults can be overridden with the `missing` parameter:
+
+[source,js]
+--------------------------------------------------
+POST /exams/_search
+{
+    "size": 0,
+    "aggs" : {
+        "weighted_grade": {
+            "weighted_avg": {
+                "value": {
+                    "field": "grade",
+                    "missing": 2
+                },
+                "weight": {
+                    "field": "weight",
+                    "missing": 3
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:exams]
+
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
index ad8ae1a6811..75f24eb7803 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
@@ -26,7 +26,7 @@ import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ArrayValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -38,7 +38,7 @@ import java.io.IOException;
 import java.util.Map;
 
 public class MatrixStatsAggregationBuilder
-    extends MultiValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MatrixStatsAggregationBuilder> {
+    extends ArrayValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MatrixStatsAggregationBuilder> {
     public static final String NAME = "matrix_stats";
 
     private MultiValueMode multiValueMode = MultiValueMode.AVG;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
index 578116d7b5e..aa19f62fedc 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
@@ -30,7 +30,7 @@ import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.MultiValuesSource.NumericMultiValuesSource;
+import org.elasticsearch.search.aggregations.support.ArrayValuesSource.NumericArrayValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -43,7 +43,7 @@ import java.util.Map;
  **/
 final class MatrixStatsAggregator extends MetricsAggregator {
     /** Multiple ValuesSource with field names */
-    private final NumericMultiValuesSource valuesSources;
+    private final NumericArrayValuesSource valuesSources;
 
     /** array of descriptive stats, per shard, needed to compute the correlation */
     ObjectArray<RunningStats> stats;
@@ -53,7 +53,7 @@ final class MatrixStatsAggregator extends MetricsAggregator {
                                  Map<String,Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
         if (valuesSources != null && !valuesSources.isEmpty()) {
-            this.valuesSources = new NumericMultiValuesSource(valuesSources, multiValueMode);
+            this.valuesSources = new NumericArrayValuesSource(valuesSources, multiValueMode);
             stats = context.bigArrays().newObjectArray(1);
         } else {
             this.valuesSources = null;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
index 2c3ac82a0c1..fb456d75bb7 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
@@ -23,7 +23,7 @@ import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ArrayValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.internal.SearchContext;
@@ -33,7 +33,7 @@ import java.util.List;
 import java.util.Map;
 
 final class MatrixStatsAggregatorFactory
-    extends MultiValuesSourceAggregatorFactory<ValuesSource.Numeric, MatrixStatsAggregatorFactory> {
+    extends ArrayValuesSourceAggregatorFactory<ValuesSource.Numeric, MatrixStatsAggregatorFactory> {
 
     private final MultiValueMode multiValueMode;
 
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
index fd13037e8f9..0f48d1855ae 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
@@ -21,14 +21,14 @@ package org.elasticsearch.search.aggregations.matrix.stats;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.MultiValueMode;
-import org.elasticsearch.search.aggregations.support.MultiValuesSourceParser.NumericValuesSourceParser;
+import org.elasticsearch.search.aggregations.support.ArrayValuesSourceParser.NumericValuesSourceParser;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Map;
 
-import static org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder.MULTIVALUE_MODE_FIELD;
+import static org.elasticsearch.search.aggregations.support.ArrayValuesSourceAggregationBuilder.MULTIVALUE_MODE_FIELD;
 
 public class MatrixStatsParser extends NumericValuesSourceParser {
 
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSource.java
similarity index 87%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java
rename to modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSource.java
index 86d1836721f..94bf68c7ae4 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSource.java
@@ -28,13 +28,13 @@ import java.util.Map;
 /**
  * Class to encapsulate a set of ValuesSource objects labeled by field name
  */
-public abstract class MultiValuesSource <VS extends ValuesSource> {
+public abstract class ArrayValuesSource<VS extends ValuesSource> {
     protected MultiValueMode multiValueMode;
     protected String[] names;
     protected VS[] values;
 
-    public static class NumericMultiValuesSource extends MultiValuesSource<ValuesSource.Numeric> {
-        public NumericMultiValuesSource(Map<String, ValuesSource.Numeric> valuesSources, MultiValueMode multiValueMode) {
+    public static class NumericArrayValuesSource extends ArrayValuesSource<ValuesSource.Numeric> {
+        public NumericArrayValuesSource(Map<String, ValuesSource.Numeric> valuesSources, MultiValueMode multiValueMode) {
             super(valuesSources, multiValueMode);
             if (valuesSources != null) {
                 this.values = valuesSources.values().toArray(new ValuesSource.Numeric[0]);
@@ -51,8 +51,8 @@ public abstract class MultiValuesSource <VS extends ValuesSource> {
         }
     }
 
-    public static class BytesMultiValuesSource extends MultiValuesSource<ValuesSource.Bytes> {
-        public BytesMultiValuesSource(Map<String, ValuesSource.Bytes> valuesSources, MultiValueMode multiValueMode) {
+    public static class BytesArrayValuesSource extends ArrayValuesSource<ValuesSource.Bytes> {
+        public BytesArrayValuesSource(Map<String, ValuesSource.Bytes> valuesSources, MultiValueMode multiValueMode) {
             super(valuesSources, multiValueMode);
             this.values = valuesSources.values().toArray(new ValuesSource.Bytes[0]);
         }
@@ -62,14 +62,14 @@ public abstract class MultiValuesSource <VS extends ValuesSource> {
         }
     }
 
-    public static class GeoPointValuesSource extends MultiValuesSource<ValuesSource.GeoPoint> {
+    public static class GeoPointValuesSource extends ArrayValuesSource<ValuesSource.GeoPoint> {
         public GeoPointValuesSource(Map<String, ValuesSource.GeoPoint> valuesSources, MultiValueMode multiValueMode) {
             super(valuesSources, multiValueMode);
             this.values = valuesSources.values().toArray(new ValuesSource.GeoPoint[0]);
         }
     }
 
-    private MultiValuesSource(Map<String, ?> valuesSources, MultiValueMode multiValueMode) {
+    private ArrayValuesSource(Map<String, ?> valuesSources, MultiValueMode multiValueMode) {
         if (valuesSources != null) {
             this.names = valuesSources.keySet().toArray(new String[0]);
         }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java
similarity index 91%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
rename to modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java
index 4cf497c9c02..eb8152e0fe0 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java
@@ -44,13 +44,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
-        extends AbstractAggregationBuilder<AB> {
+public abstract class ArrayValuesSourceAggregationBuilder<VS extends ValuesSource, AB extends ArrayValuesSourceAggregationBuilder<VS, AB>>
+    extends AbstractAggregationBuilder<AB> {
 
     public static final ParseField MULTIVALUE_MODE_FIELD = new ParseField("mode");
 
-    public abstract static class LeafOnly<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
-            extends MultiValuesSourceAggregationBuilder<VS, AB> {
+    public abstract static class LeafOnly<VS extends ValuesSource, AB extends ArrayValuesSourceAggregationBuilder<VS, AB>>
+        extends ArrayValuesSourceAggregationBuilder<VS, AB> {
 
         protected LeafOnly(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
             super(name, valuesSourceType, targetValueType);
@@ -94,7 +94,7 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
     private Object missing = null;
     private Map<String, Object> missingMap = Collections.emptyMap();
 
-    protected MultiValuesSourceAggregationBuilder(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+    protected ArrayValuesSourceAggregationBuilder(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
         super(name);
         if (valuesSourceType == null) {
             throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]");
@@ -103,7 +103,7 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
         this.targetValueType = targetValueType;
     }
 
-    protected MultiValuesSourceAggregationBuilder(MultiValuesSourceAggregationBuilder<VS, AB> clone,
+    protected ArrayValuesSourceAggregationBuilder(ArrayValuesSourceAggregationBuilder<VS, AB> clone,
                                                   Builder factoriesBuilder, Map<String, Object> metaData) {
         super(clone, factoriesBuilder, metaData);
         this.valuesSourceType = clone.valuesSourceType;
@@ -115,7 +115,7 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
         this.missing = clone.missing;
     }
 
-    protected MultiValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType)
+    protected ArrayValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType)
         throws IOException {
         super(in);
         assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType";
@@ -124,7 +124,7 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
         read(in);
     }
 
-    protected MultiValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+    protected ArrayValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
         super(in);
         assert serializeTargetValueType() : "Wrong read constructor called for subclass that serializes its targetValueType";
         this.valuesSourceType = valuesSourceType;
@@ -239,10 +239,10 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
     }
 
     @Override
-    protected final MultiValuesSourceAggregatorFactory<VS, ?> doBuild(SearchContext context, AggregatorFactory<?> parent,
-            AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
+    protected final ArrayValuesSourceAggregatorFactory<VS, ?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+                                                                      AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         Map<String, ValuesSourceConfig<VS>> configs = resolveConfig(context);
-        MultiValuesSourceAggregatorFactory<VS, ?> factory = innerBuild(context, configs, parent, subFactoriesBuilder);
+        ArrayValuesSourceAggregatorFactory<VS, ?> factory = innerBuild(context, configs, parent, subFactoriesBuilder);
         return factory;
     }
 
@@ -255,9 +255,10 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
         return configs;
     }
 
-    protected abstract MultiValuesSourceAggregatorFactory<VS, ?> innerBuild(SearchContext context,
-            Map<String, ValuesSourceConfig<VS>> configs, AggregatorFactory<?> parent,
-            AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
+    protected abstract ArrayValuesSourceAggregatorFactory<VS, ?> innerBuild(SearchContext context,
+                                                                Map<String, ValuesSourceConfig<VS>> configs,
+                                                                AggregatorFactory<?> parent,
+                                                                AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
 
     public ValuesSourceConfig<VS> config(SearchContext context, String field, Script script) {
 
@@ -355,14 +356,14 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
     @Override
     protected final int doHashCode() {
         return Objects.hash(fields, format, missing, targetValueType, valueType, valuesSourceType,
-                innerHashCode());
+            innerHashCode());
     }
 
     protected abstract int innerHashCode();
 
     @Override
     protected final boolean doEquals(Object obj) {
-        MultiValuesSourceAggregationBuilder<?, ?> other = (MultiValuesSourceAggregationBuilder<?, ?>) obj;
+        ArrayValuesSourceAggregationBuilder<?, ?> other = (ArrayValuesSourceAggregationBuilder<?, ?>) obj;
         if (!Objects.equals(fields, other.fields))
             return false;
         if (!Objects.equals(format, other.format))
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java
similarity index 78%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
rename to modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java
index 7d5c56a571b..ce8eeecd190 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java
@@ -30,14 +30,15 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-public abstract class MultiValuesSourceAggregatorFactory<VS extends ValuesSource, AF extends MultiValuesSourceAggregatorFactory<VS, AF>>
-        extends AggregatorFactory<AF> {
+public abstract class ArrayValuesSourceAggregatorFactory<VS extends ValuesSource, AF extends ArrayValuesSourceAggregatorFactory<VS, AF>>
+    extends AggregatorFactory<AF> {
 
     protected Map<String, ValuesSourceConfig<VS>> configs;
 
-    public MultiValuesSourceAggregatorFactory(String name, Map<String, ValuesSourceConfig<VS>> configs,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
-            Map<String, Object> metaData) throws IOException {
+    public ArrayValuesSourceAggregatorFactory(String name, Map<String, ValuesSourceConfig<VS>> configs,
+                                              SearchContext context, AggregatorFactory<?> parent,
+                                              AggregatorFactories.Builder subFactoriesBuilder,
+                                              Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metaData);
         this.configs = configs;
     }
@@ -63,6 +64,7 @@ public abstract class MultiValuesSourceAggregatorFactory<VS extends ValuesSource
                                                  Map<String, Object> metaData) throws IOException;
 
     protected abstract Aggregator doCreateInternal(Map<String, VS> valuesSources, Aggregator parent, boolean collectsFromSingleBucket,
-        List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException;
+                                                   List<PipelineAggregator> pipelineAggregators,
+                                                   Map<String, Object> metaData) throws IOException;
 
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java
similarity index 86%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
rename to modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java
index 22a90b552d9..1100884cf8a 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java
@@ -33,30 +33,30 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-public abstract class MultiValuesSourceParser<VS extends ValuesSource> implements Aggregator.Parser {
+public abstract class ArrayValuesSourceParser<VS extends ValuesSource> implements Aggregator.Parser {
 
-    public abstract static class AnyValuesSourceParser extends MultiValuesSourceParser<ValuesSource> {
+    public abstract static class AnyValuesSourceParser extends ArrayValuesSourceParser<ValuesSource> {
 
         protected AnyValuesSourceParser(boolean formattable) {
             super(formattable, ValuesSourceType.ANY, null);
         }
     }
 
-    public abstract static class NumericValuesSourceParser extends MultiValuesSourceParser<ValuesSource.Numeric> {
+    public abstract static class NumericValuesSourceParser extends ArrayValuesSourceParser<ValuesSource.Numeric> {
 
         protected NumericValuesSourceParser(boolean formattable) {
             super(formattable, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         }
     }
 
-    public abstract static class BytesValuesSourceParser extends MultiValuesSourceParser<ValuesSource.Bytes> {
+    public abstract static class BytesValuesSourceParser extends ArrayValuesSourceParser<ValuesSource.Bytes> {
 
         protected BytesValuesSourceParser(boolean formattable) {
             super(formattable, ValuesSourceType.BYTES, ValueType.STRING);
         }
     }
 
-    public abstract static class GeoPointValuesSourceParser extends MultiValuesSourceParser<ValuesSource.GeoPoint> {
+    public abstract static class GeoPointValuesSourceParser extends ArrayValuesSourceParser<ValuesSource.GeoPoint> {
 
         protected GeoPointValuesSourceParser(boolean formattable) {
             super(formattable, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
@@ -67,15 +67,15 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
     private ValuesSourceType valuesSourceType = null;
     private ValueType targetValueType = null;
 
-    private MultiValuesSourceParser(boolean formattable, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+    private ArrayValuesSourceParser(boolean formattable, ValuesSourceType valuesSourceType, ValueType targetValueType) {
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = targetValueType;
         this.formattable = formattable;
     }
 
     @Override
-    public final MultiValuesSourceAggregationBuilder<VS, ?> parse(String aggregationName, XContentParser parser)
-            throws IOException {
+    public final ArrayValuesSourceAggregationBuilder<VS, ?> parse(String aggregationName, XContentParser parser)
+        throws IOException {
 
         List<String> fields = null;
         ValueType valueType = null;
@@ -98,7 +98,7 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
                             "Multi-field aggregations do not support scripts.");
                 } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
-                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (CommonFields.MISSING.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -113,7 +113,7 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
 
                 } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
-                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
                 if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -127,21 +127,21 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
                             fields.add(parser.text());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
-                                    "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                                "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                         }
                     }
                 } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
-                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
             } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                 throw new ParsingException(parser.getTokenLocation(),
-                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                    "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
             }
         }
 
-        MultiValuesSourceAggregationBuilder<VS, ?> factory = createFactory(aggregationName, this.valuesSourceType, this.targetValueType,
-                otherOptions);
+        ArrayValuesSourceAggregationBuilder<VS, ?> factory = createFactory(aggregationName, this.valuesSourceType, this.targetValueType,
+            otherOptions);
         if (fields != null) {
             factory.fields(fields);
         }
@@ -182,7 +182,7 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
     /**
      * Creates a {@link ValuesSourceAggregationBuilder} from the information
      * gathered by the subclass. Options parsed in
-     * {@link MultiValuesSourceParser} itself will be added to the factory
+     * {@link ArrayValuesSourceParser} itself will be added to the factory
      * after it has been returned by this method.
      *
      * @param aggregationName
@@ -197,11 +197,13 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
      *            method
      * @return the created factory
      */
-    protected abstract MultiValuesSourceAggregationBuilder<VS, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-        ValueType targetValueType, Map<ParseField, Object> otherOptions);
+    protected abstract ArrayValuesSourceAggregationBuilder<VS, ?> createFactory(String aggregationName,
+                                                                                ValuesSourceType valuesSourceType,
+                                                                                ValueType targetValueType,
+                                                                                Map<ParseField, Object> otherOptions);
 
     /**
-     * Allows subclasses of {@link MultiValuesSourceParser} to parse extra
+     * Allows subclasses of {@link ArrayValuesSourceParser} to parse extra
      * parameters and store them in a {@link Map} which will later be passed to
      * {@link #createFactory(String, ValuesSourceType, ValueType, Map)}.
      *
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml
new file mode 100644
index 00000000000..bccc9619233
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml
@@ -0,0 +1,71 @@
+setup:
+  - do:
+      indices.create:
+          index: test_1
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              doc:
+                properties:
+                  int_field:
+                     type : integer
+                  double_field:
+                     type : double
+                  string_field:
+                     type: keyword
+
+  - do:
+       bulk:
+         refresh: true
+         body:
+           - index:
+               _index: test_1
+               _type:  doc
+               _id:    1
+           - int_field: 1
+             double_field: 1.0
+           - index:
+               _index: test_1
+               _type:  doc
+               _id:    2
+           - int_field: 2
+             double_field: 2.0
+           - index:
+               _index: test_1
+               _type:  doc
+               _id:    3
+           - int_field: 3
+             double_field: 3.0
+           - index:
+               _index: test_1
+               _type:  doc
+               _id:    4
+           - int_field: 4
+             double_field: 4.0
+
+---
+"Basic test":
+
+  - do:
+      search:
+        body:
+          aggs:
+            the_int_avg:
+              weighted_avg:
+                value:
+                  field: "int_field"
+                weight:
+                  field: "int_field"
+            the_double_avg:
+              weighted_avg:
+                value:
+                  field: "double_field"
+                weight:
+                  field: "double_field"
+
+  - match: { hits.total: 4 }
+  - length: { hits.hits: 4 }
+  - match: { aggregations.the_int_avg.value: 3.0 }
+  - match: { aggregations.the_double_avg.value: 3.0 }
+
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index efef1aeb04f..99b47cf83e2 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -181,6 +181,8 @@ import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits;
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.weighted_avg.InternalWeightedAvg;
+import org.elasticsearch.search.aggregations.metrics.weighted_avg.WeightedAvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
@@ -335,6 +337,8 @@ public class SearchModule {
     private void registerAggregations(List<SearchPlugin> plugins) {
         registerAggregation(new AggregationSpec(AvgAggregationBuilder.NAME, AvgAggregationBuilder::new, AvgAggregationBuilder::parse)
                 .addResultReader(InternalAvg::new));
+        registerAggregation(new AggregationSpec(WeightedAvgAggregationBuilder.NAME, WeightedAvgAggregationBuilder::new,
+            WeightedAvgAggregationBuilder::parse).addResultReader(InternalWeightedAvg::new));
         registerAggregation(new AggregationSpec(SumAggregationBuilder.NAME, SumAggregationBuilder::new, SumAggregationBuilder::parse)
                 .addResultReader(InternalSum::new));
         registerAggregation(new AggregationSpec(MinAggregationBuilder.NAME, MinAggregationBuilder::new, MinAggregationBuilder::parse)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
index 26d8bb1a1bd..b4e416f4d77 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
@@ -82,6 +82,7 @@ import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.weighted_avg.WeightedAvgAggregationBuilder;
 
 import java.util.Map;
 
@@ -107,6 +108,13 @@ public class AggregationBuilders {
         return new AvgAggregationBuilder(name);
     }
 
+    /**
+     * Create a new {@link Avg} aggregation with the given name.
+     */
+    public static WeightedAvgAggregationBuilder weightedAvg(String name) {
+        return new WeightedAvgAggregationBuilder(name);
+    }
+
     /**
      * Create a new {@link Max} aggregation with the given name.
      */
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/InternalWeightedAvg.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/InternalWeightedAvg.java
new file mode 100644
index 00000000000..9ad1a1df78a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/InternalWeightedAvg.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.weighted_avg;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class InternalWeightedAvg extends InternalNumericMetricsAggregation.SingleValue implements WeightedAvg {
+    private final double sum;
+    private final double weight;
+
+    public InternalWeightedAvg(String name, double sum, double weight, DocValueFormat format, List<PipelineAggregator> pipelineAggregators,
+                               Map<String, Object> metaData) {
+        super(name, pipelineAggregators, metaData);
+        this.sum = sum;
+        this.weight = weight;
+        this.format = format;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public InternalWeightedAvg(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        sum = in.readDouble();
+        weight = in.readDouble();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeDouble(sum);
+        out.writeDouble(weight);
+    }
+
+    @Override
+    public double value() {
+        return getValue();
+    }
+
+    @Override
+    public double getValue() {
+        return sum / weight;
+    }
+
+    double getSum() {
+        return sum;
+    }
+
+    double getWeight() {
+        return weight;
+    }
+
+    DocValueFormat getFormatter() {
+        return format;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return WeightedAvgAggregationBuilder.NAME;
+    }
+
+    @Override
+    public InternalWeightedAvg doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        double weight = 0;
+        double sum = 0;
+        double sumCompensation = 0;
+        double weightCompensation = 0;
+        // Compute the sum of double values with Kahan summation algorithm which is more
+        // accurate than naive summation.
+        for (InternalAggregation aggregation : aggregations) {
+            InternalWeightedAvg avg = (InternalWeightedAvg) aggregation;
+            // If the weight is Inf or NaN, just add it to the running tally to "convert" to
+            // Inf/NaN.  This keeps the behavior bwc from before kahan summing
+            if (Double.isFinite(avg.weight) == false) {
+                weight += avg.weight;
+            } else if (Double.isFinite(weight)) {
+                double corrected = avg.weight - weightCompensation;
+                double newWeight = weight + corrected;
+                weightCompensation = (newWeight - weight) - corrected;
+                weight = newWeight;
+            }
+            // If the avg is Inf or NaN, just add it to the running tally to "convert" to
+            // Inf/NaN.  This keeps the behavior bwc from before kahan summing
+            if (Double.isFinite(avg.sum) == false) {
+                sum += avg.sum;
+            } else if (Double.isFinite(sum)) {
+                double corrected = avg.sum - sumCompensation;
+                double newSum = sum + corrected;
+                sumCompensation = (newSum - sum) - corrected;
+                sum = newSum;
+            }
+        }
+        return new InternalWeightedAvg(getName(), sum, weight, format, pipelineAggregators(), getMetaData());
+    }
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(CommonFields.VALUE.getPreferredName(), weight != 0 ? getValue() : null);
+        if (weight != 0 && format != DocValueFormat.RAW) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(getValue()));
+        }
+        return builder;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(sum, weight, format.getWriteableName());
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        InternalWeightedAvg other = (InternalWeightedAvg) obj;
+        return Objects.equals(sum, other.sum) &&
+                Objects.equals(weight, other.weight) &&
+                Objects.equals(format.getWriteableName(), other.format.getWriteableName());
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/ParsedWeightedAvg.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/ParsedWeightedAvg.java
new file mode 100644
index 00000000000..dcda79ce33e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/ParsedWeightedAvg.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.weighted_avg;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
+
+import java.io.IOException;
+
+public class ParsedWeightedAvg extends ParsedSingleValueNumericMetricsAggregation implements WeightedAvg {
+
+    @Override
+    public double getValue() {
+        return value();
+    }
+
+    @Override
+    public String getType() {
+        return WeightedAvgAggregationBuilder.NAME;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        // InternalWeightedAvg renders value only if the avg normalizer (count) is not 0.
+        // We parse back `null` as Double.POSITIVE_INFINITY so we check for that value here to get the same xContent output
+        boolean hasValue = value != Double.POSITIVE_INFINITY;
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? value : null);
+        if (hasValue && valueAsString != null) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString);
+        }
+        return builder;
+    }
+
+    private static final ObjectParser<ParsedWeightedAvg, Void> PARSER
+        = new ObjectParser<>(ParsedWeightedAvg.class.getSimpleName(), true, ParsedWeightedAvg::new);
+
+    static {
+        declareSingleValueFields(PARSER, Double.POSITIVE_INFINITY);
+    }
+
+    public static ParsedWeightedAvg fromXContent(XContentParser parser, final String name) {
+        ParsedWeightedAvg avg = PARSER.apply(parser, null);
+        avg.setName(name);
+        return avg;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvg.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvg.java
new file mode 100644
index 00000000000..7af48f677c1
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvg.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.weighted_avg;
+
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
+
+/**
+ * An aggregation that computes the average of the values in the current bucket.
+ */
+public interface WeightedAvg extends NumericMetricsAggregation.SingleValue {
+
+    /**
+     * The average value.
+     */
+    double getValue();
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilder.java
new file mode 100644
index 00000000000..be06f792a5e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilder.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.weighted_avg;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceParseHelper;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class WeightedAvgAggregationBuilder extends MultiValuesSourceAggregationBuilder.LeafOnly<Numeric, WeightedAvgAggregationBuilder> {
+    public static final String NAME = "weighted_avg";
+    public static final ParseField VALUE_FIELD = new ParseField("value");
+    public static final ParseField WEIGHT_FIELD = new ParseField("weight");
+
+    private static final ObjectParser<WeightedAvgAggregationBuilder, Void> PARSER;
+    static {
+        PARSER = new ObjectParser<>(WeightedAvgAggregationBuilder.NAME);
+        MultiValuesSourceParseHelper.declareCommon(PARSER, true, ValueType.NUMERIC);
+        MultiValuesSourceParseHelper.declareField(VALUE_FIELD.getPreferredName(), PARSER, true, false);
+        MultiValuesSourceParseHelper.declareField(WEIGHT_FIELD.getPreferredName(), PARSER, true, false);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
+        return PARSER.parse(parser, new WeightedAvgAggregationBuilder(aggregationName), null);
+    }
+
+    public WeightedAvgAggregationBuilder(String name) {
+        super(name, ValueType.NUMERIC);
+    }
+
+    public WeightedAvgAggregationBuilder(WeightedAvgAggregationBuilder clone, Builder factoriesBuilder, Map<String, Object> metaData) {
+        super(clone, factoriesBuilder, metaData);
+    }
+
+    public WeightedAvgAggregationBuilder value(MultiValuesSourceFieldConfig valueConfig) {
+        valueConfig = Objects.requireNonNull(valueConfig, "Configuration for field [" + VALUE_FIELD + "] cannot be null");
+        field(VALUE_FIELD.getPreferredName(), valueConfig);
+        return this;
+    }
+
+    public WeightedAvgAggregationBuilder weight(MultiValuesSourceFieldConfig weightConfig) {
+        weightConfig = Objects.requireNonNull(weightConfig, "Configuration for field [" + WEIGHT_FIELD + "] cannot be null");
+        field(WEIGHT_FIELD.getPreferredName(), weightConfig);
+        return this;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public WeightedAvgAggregationBuilder(StreamInput in) throws IOException {
+        super(in, ValueType.NUMERIC);
+    }
+
+    @Override
+    protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, Object> metaData) {
+        return new WeightedAvgAggregationBuilder(this, factoriesBuilder, metaData);
+    }
+
+    @Override
+    protected void innerWriteTo(StreamOutput out) {
+        // Do nothing, no extra state to write to stream
+    }
+
+    @Override
+    protected MultiValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context,
+                                                                    Map<String, ValuesSourceConfig<Numeric>> configs,
+                                                                    DocValueFormat format,
+                                                                    AggregatorFactory<?> parent,
+                                                                    Builder subFactoriesBuilder) throws IOException {
+        return new WeightedAvgAggregatorFactory(name, configs, format, context, parent, subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        return builder;
+    }
+
+    @Override
+    protected int innerHashCode() {
+        return 0;
+    }
+
+    @Override
+    protected boolean innerEquals(Object obj) {
+        return true;
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregator.java
new file mode 100644
index 00000000000..7a34fe6df4a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregator.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.weighted_avg;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.DoubleArray;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.MultiValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.search.aggregations.metrics.weighted_avg.WeightedAvgAggregationBuilder.VALUE_FIELD;
+import static org.elasticsearch.search.aggregations.metrics.weighted_avg.WeightedAvgAggregationBuilder.WEIGHT_FIELD;
+
+public class WeightedAvgAggregator extends NumericMetricsAggregator.SingleValue {
+
+    private final MultiValuesSource.NumericMultiValuesSource valuesSources;
+
+    private DoubleArray weights;
+    private DoubleArray sums;
+    private DoubleArray sumCompensations;
+    private DoubleArray weightCompensations;
+    private DocValueFormat format;
+
+    public WeightedAvgAggregator(String name, MultiValuesSource.NumericMultiValuesSource valuesSources, DocValueFormat format,
+                                 SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                                 Map<String, Object> metaData) throws IOException {
+        super(name, context, parent, pipelineAggregators, metaData);
+        this.valuesSources = valuesSources;
+        this.format = format;
+        if (valuesSources != null) {
+            final BigArrays bigArrays = context.bigArrays();
+            weights = bigArrays.newDoubleArray(1, true);
+            sums = bigArrays.newDoubleArray(1, true);
+            sumCompensations = bigArrays.newDoubleArray(1, true);
+            weightCompensations = bigArrays.newDoubleArray(1, true);
+        }
+    }
+
+    @Override
+    public boolean needsScores() {
+        return valuesSources != null && valuesSources.needsScores();
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
+            final LeafBucketCollector sub) throws IOException {
+        if (valuesSources == null) {
+            return LeafBucketCollector.NO_OP_COLLECTOR;
+        }
+        final BigArrays bigArrays = context.bigArrays();
+        final SortedNumericDoubleValues docValues = valuesSources.getField(VALUE_FIELD.getPreferredName(), ctx);
+        final SortedNumericDoubleValues docWeights = valuesSources.getField(WEIGHT_FIELD.getPreferredName(), ctx);
+
+        return new LeafBucketCollectorBase(sub, docValues) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                weights = bigArrays.grow(weights, bucket + 1);
+                sums = bigArrays.grow(sums, bucket + 1);
+                sumCompensations = bigArrays.grow(sumCompensations, bucket + 1);
+                weightCompensations = bigArrays.grow(weightCompensations, bucket + 1);
+
+                if (docValues.advanceExact(doc) && docWeights.advanceExact(doc)) {
+                    if (docWeights.docValueCount() > 1) {
+                        throw new AggregationExecutionException("Encountered more than one weight for a " +
+                            "single document. Use a script to combine multiple weights-per-doc into a single value.");
+                    }
+                    // There should always be one weight if advanceExact lands us here, either
+                    // a real weight or a `missing` weight
+                    assert docWeights.docValueCount() == 1;
+                    final double weight = docWeights.nextValue();
+
+                    final int numValues = docValues.docValueCount();
+                    assert numValues > 0;
+
+                    for (int i = 0; i < numValues; i++) {
+                        kahanSum(docValues.nextValue() * weight, sums, sumCompensations, bucket);
+                        kahanSum(weight, weights, weightCompensations, bucket);
+                    }
+                }
+            }
+        };
+    }
+
+    private static void kahanSum(double value, DoubleArray values, DoubleArray compensations, long bucket) {
+        // Compute the sum of double values with Kahan summation algorithm which is more
+        // accurate than naive summation.
+        double sum = values.get(bucket);
+        double compensation = compensations.get(bucket);
+
+        if (Double.isFinite(value) == false) {
+            sum += value;
+        } else if (Double.isFinite(sum)) {
+            double corrected = value - compensation;
+            double newSum = sum + corrected;
+            compensation = (newSum - sum) - corrected;
+            sum = newSum;
+        }
+        values.set(bucket, sum);
+        compensations.set(bucket, compensation);
+    }
+
+    @Override
+    public double metric(long owningBucketOrd) {
+        if (valuesSources == null || owningBucketOrd >= sums.size()) {
+            return Double.NaN;
+        }
+        return sums.get(owningBucketOrd) / weights.get(owningBucketOrd);
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long bucket) {
+        if (valuesSources == null || bucket >= sums.size()) {
+            return buildEmptyAggregation();
+        }
+        return new InternalWeightedAvg(name, sums.get(bucket), weights.get(bucket), format, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        return new InternalWeightedAvg(name, 0.0, 0L, format, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public void doClose() {
+        Releasables.close(weights, sums, sumCompensations, weightCompensations);
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregatorFactory.java
new file mode 100644
index 00000000000..c7aab73af28
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregatorFactory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.weighted_avg;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.MultiValuesSource;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class WeightedAvgAggregatorFactory extends MultiValuesSourceAggregatorFactory<Numeric, WeightedAvgAggregatorFactory> {
+
+    public WeightedAvgAggregatorFactory(String name,  Map<String, ValuesSourceConfig<Numeric>> configs,
+                                        DocValueFormat format, SearchContext context, AggregatorFactory<?> parent,
+                                        AggregatorFactories.Builder subFactoriesBuilder,
+                                        Map<String, Object> metaData) throws IOException {
+        super(name, configs, format, context, parent, subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+            throws IOException {
+        return new WeightedAvgAggregator(name, null, format, context, parent, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Aggregator doCreateInternal(Map<String, ValuesSourceConfig<Numeric>> configs, DocValueFormat format,
+                                          Aggregator parent, boolean collectsFromSingleBucket,
+                                          List<PipelineAggregator> pipelineAggregators,
+                                          Map<String, Object> metaData) throws IOException {
+        MultiValuesSource.NumericMultiValuesSource numericMultiVS
+            = new MultiValuesSource.NumericMultiValuesSource(configs, context.getQueryShardContext());
+        if (numericMultiVS.areValuesSourcesEmpty()) {
+            return createUnmapped(parent, pipelineAggregators, metaData);
+        }
+        return new WeightedAvgAggregator(name, numericMultiVS, format, context, parent, pipelineAggregators, metaData);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java
new file mode 100644
index 00000000000..9ceecd75dea
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.support;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.query.QueryShardContext;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Class to encapsulate a set of ValuesSource objects labeled by field name
+ */
+public abstract class MultiValuesSource <VS extends ValuesSource> {
+    protected Map<String, VS> values;
+
+    public static class NumericMultiValuesSource extends MultiValuesSource<ValuesSource.Numeric> {
+        public NumericMultiValuesSource(Map<String, ValuesSourceConfig<ValuesSource.Numeric>> valuesSourceConfigs,
+                                        QueryShardContext context) throws IOException {
+            values = new HashMap<>(valuesSourceConfigs.size());
+            for (Map.Entry<String, ValuesSourceConfig<ValuesSource.Numeric>> entry : valuesSourceConfigs.entrySet()) {
+                values.put(entry.getKey(), entry.getValue().toValuesSource(context));
+            }
+        }
+
+        public SortedNumericDoubleValues getField(String fieldName, LeafReaderContext ctx) throws IOException {
+            ValuesSource.Numeric value = values.get(fieldName);
+            if (value == null) {
+                throw new IllegalArgumentException("Could not find field name [" + fieldName + "] in multiValuesSource");
+            }
+            return value.doubleValues(ctx);
+        }
+    }
+
+    public static class BytesMultiValuesSource extends MultiValuesSource<ValuesSource.Bytes> {
+        public BytesMultiValuesSource(Map<String, ValuesSourceConfig<ValuesSource.Bytes>> valuesSourceConfigs,
+                                      QueryShardContext context) throws IOException {
+            values = new HashMap<>(valuesSourceConfigs.size());
+            for (Map.Entry<String, ValuesSourceConfig<ValuesSource.Bytes>> entry : valuesSourceConfigs.entrySet()) {
+                values.put(entry.getKey(), entry.getValue().toValuesSource(context));
+            }
+        }
+
+        public Object getField(String fieldName, LeafReaderContext ctx) throws IOException {
+            ValuesSource.Bytes value = values.get(fieldName);
+            if (value == null) {
+                throw new IllegalArgumentException("Could not find field name [" + fieldName + "] in multiValuesSource");
+            }
+            return value.bytesValues(ctx);
+        }
+    }
+
+    public static class GeoPointValuesSource extends MultiValuesSource<ValuesSource.GeoPoint> {
+        public GeoPointValuesSource(Map<String, ValuesSourceConfig<ValuesSource.GeoPoint>> valuesSourceConfigs,
+                                    QueryShardContext context) throws IOException {
+            values = new HashMap<>(valuesSourceConfigs.size());
+            for (Map.Entry<String, ValuesSourceConfig<ValuesSource.GeoPoint>> entry : valuesSourceConfigs.entrySet()) {
+                values.put(entry.getKey(), entry.getValue().toValuesSource(context));
+            }
+        }
+    }
+
+    public boolean needsScores() {
+        return values.values().stream().anyMatch(ValuesSource::needsScores);
+    }
+
+    public String[] fieldNames() {
+        return values.keySet().toArray(new String[0]);
+    }
+
+    public boolean areValuesSourcesEmpty() {
+        return values.values().stream().allMatch(Objects::isNull);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
new file mode 100644
index 00000000000..fee685346ec
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationInitializationException;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Similar to {@link ValuesSourceAggregationBuilder}, except it references multiple ValuesSources (e.g. so that an aggregation
+ * can pull values from multiple fields).
+ *
+ * A limitation of this class is that all the ValuesSource's being refereenced must be of the same type.
+ */
+public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
+        extends AbstractAggregationBuilder<AB> {
+
+
+    public abstract static class LeafOnly<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
+            extends MultiValuesSourceAggregationBuilder<VS, AB> {
+
+        protected LeafOnly(String name, ValueType targetValueType) {
+            super(name, targetValueType);
+        }
+
+        protected LeafOnly(LeafOnly<VS, AB> clone, Builder factoriesBuilder, Map<String, Object> metaData) {
+            super(clone, factoriesBuilder, metaData);
+            if (factoriesBuilder.count() > 0) {
+                throw new AggregationInitializationException("Aggregator [" + name + "] of type ["
+                    + getType() + "] cannot accept sub-aggregations");
+            }
+        }
+
+        /**
+         * Read from a stream that does not serialize its targetValueType. This should be used by most subclasses.
+         */
+        protected LeafOnly(StreamInput in, ValueType targetValueType) throws IOException {
+            super(in, targetValueType);
+        }
+
+        @Override
+        public AB subAggregations(Builder subFactories) {
+            throw new AggregationInitializationException("Aggregator [" + name + "] of type [" +
+                getType() + "] cannot accept sub-aggregations");
+        }
+    }
+
+
+
+    private Map<String, MultiValuesSourceFieldConfig> fields = new HashMap<>();
+    private final ValueType targetValueType;
+    private ValueType valueType = null;
+    private String format = null;
+
+    protected MultiValuesSourceAggregationBuilder(String name, ValueType targetValueType) {
+        super(name);
+        this.targetValueType = targetValueType;
+    }
+
+    protected MultiValuesSourceAggregationBuilder(MultiValuesSourceAggregationBuilder<VS, AB> clone,
+                                                  Builder factoriesBuilder, Map<String, Object> metaData) {
+        super(clone, factoriesBuilder, metaData);
+
+        this.fields = new HashMap<>(clone.fields);
+        this.targetValueType = clone.targetValueType;
+        this.valueType = clone.valueType;
+        this.format = clone.format;
+    }
+
+    protected MultiValuesSourceAggregationBuilder(StreamInput in, ValueType targetValueType)
+        throws IOException {
+        super(in);
+        assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType";
+        this.targetValueType = targetValueType;
+        read(in);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    @SuppressWarnings("unchecked")
+    private void read(StreamInput in) throws IOException {
+        fields = in.readMap(StreamInput::readString, MultiValuesSourceFieldConfig::new);
+        valueType = in.readOptionalWriteable(ValueType::readFromStream);
+        format = in.readOptionalString();
+    }
+
+    @Override
+    protected final void doWriteTo(StreamOutput out) throws IOException {
+        if (serializeTargetValueType()) {
+            out.writeOptionalWriteable(targetValueType);
+        }
+        out.writeMap(fields, StreamOutput::writeString, (o, value) -> value.writeTo(o));
+        out.writeOptionalWriteable(valueType);
+        out.writeOptionalString(format);
+        innerWriteTo(out);
+    }
+
+    /**
+     * Write subclass' state to the stream
+     */
+    protected abstract void innerWriteTo(StreamOutput out) throws IOException;
+
+    @SuppressWarnings("unchecked")
+    protected AB field(String propertyName, MultiValuesSourceFieldConfig config) {
+        if (config == null) {
+            throw new IllegalArgumentException("[config] must not be null: [" + name + "]");
+        }
+        this.fields.put(propertyName, config);
+        return (AB) this;
+    }
+
+    public Map<String, MultiValuesSourceFieldConfig> fields() {
+        return fields;
+    }
+
+    /**
+     * Sets the {@link ValueType} for the value produced by this aggregation
+     */
+    @SuppressWarnings("unchecked")
+    public AB valueType(ValueType valueType) {
+        if (valueType == null) {
+            throw new IllegalArgumentException("[valueType] must not be null: [" + name + "]");
+        }
+        this.valueType = valueType;
+        return (AB) this;
+    }
+
+    /**
+     * Gets the {@link ValueType} for the value produced by this aggregation
+     */
+    public ValueType valueType() {
+        return valueType;
+    }
+
+    /**
+     * Sets the format to use for the output of the aggregation.
+     */
+    @SuppressWarnings("unchecked")
+    public AB format(String format) {
+        if (format == null) {
+            throw new IllegalArgumentException("[format] must not be null: [" + name + "]");
+        }
+        this.format = format;
+        return (AB) this;
+    }
+
+    /**
+     * Gets the format to use for the output of the aggregation.
+     */
+    public String format() {
+        return format;
+    }
+
+    @Override
+    protected final MultiValuesSourceAggregatorFactory<VS, ?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+            AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
+        ValueType finalValueType = this.valueType != null ? this.valueType : targetValueType;
+
+        Map<String, ValuesSourceConfig<VS>> configs = new HashMap<>(fields.size());
+        fields.forEach((key, value) -> {
+            ValuesSourceConfig<VS> config = ValuesSourceConfig.resolve(context.getQueryShardContext(), finalValueType,
+                value.getFieldName(), value.getScript(), value.getMissing(), value.getTimeZone(), format);
+            configs.put(key, config);
+        });
+        DocValueFormat docValueFormat = resolveFormat(format, finalValueType);
+        return innerBuild(context, configs, docValueFormat, parent, subFactoriesBuilder);
+    }
+
+
+    private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType) {
+        if (valueType == null) {
+            return DocValueFormat.RAW; // we can't figure it out
+        }
+        DocValueFormat valueFormat = valueType.defaultFormat;
+        if (valueFormat instanceof DocValueFormat.Decimal && format != null) {
+            valueFormat = new DocValueFormat.Decimal(format);
+        }
+        return valueFormat;
+    }
+
+    protected abstract MultiValuesSourceAggregatorFactory<VS, ?> innerBuild(SearchContext context,
+        Map<String, ValuesSourceConfig<VS>> configs, DocValueFormat format, AggregatorFactory<?> parent,
+        AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
+
+
+    /**
+     * Should this builder serialize its targetValueType? Defaults to false. All subclasses that override this to true
+     * should use the three argument read constructor rather than the four argument version.
+     */
+    protected boolean serializeTargetValueType() {
+        return false;
+    }
+
+    @Override
+    public final XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (fields != null) {
+            builder.field(CommonFields.FIELDS.getPreferredName(), fields);
+        }
+        if (format != null) {
+            builder.field(CommonFields.FORMAT.getPreferredName(), format);
+        }
+        if (valueType != null) {
+            builder.field(CommonFields.VALUE_TYPE.getPreferredName(), valueType.getPreferredName());
+        }
+        doXContentBody(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
+
+    @Override
+    protected final int doHashCode() {
+        return Objects.hash(fields, format, targetValueType, valueType, innerHashCode());
+    }
+
+    protected abstract int innerHashCode();
+
+    @Override
+    protected final boolean doEquals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        MultiValuesSourceAggregationBuilder that = (MultiValuesSourceAggregationBuilder) other;
+
+        return Objects.equals(this.fields, that.fields)
+            && Objects.equals(this.format, that.format)
+            && Objects.equals(this.valueType, that.valueType);
+    }
+
+    protected abstract boolean innerEquals(Object obj);
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
new file mode 100644
index 00000000000..5de8fbd7561
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public abstract class MultiValuesSourceAggregatorFactory<VS extends ValuesSource, AF extends MultiValuesSourceAggregatorFactory<VS, AF>>
+        extends AggregatorFactory<AF> {
+
+    protected final Map<String, ValuesSourceConfig<VS>> configs;
+    protected final DocValueFormat format;
+
+    public MultiValuesSourceAggregatorFactory(String name, Map<String, ValuesSourceConfig<VS>> configs,
+                                              DocValueFormat format, SearchContext context,
+                                              AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+                                              Map<String, Object> metaData) throws IOException {
+        super(name, context, parent, subFactoriesBuilder, metaData);
+        this.configs = configs;
+        this.format = format;
+    }
+
+    @Override
+    public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators,
+                                     Map<String, Object> metaData) throws IOException {
+
+        return doCreateInternal(configs, format, parent, collectsFromSingleBucket,
+            pipelineAggregators, metaData);
+    }
+
+    protected abstract Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                                                 Map<String, Object> metaData) throws IOException;
+
+    protected abstract Aggregator doCreateInternal(Map<String, ValuesSourceConfig<VS>> configs,
+                                                   DocValueFormat format, Aggregator parent, boolean collectsFromSingleBucket,
+                                                   List<PipelineAggregator> pipelineAggregators,
+                                                   Map<String, Object> metaData) throws IOException;
+
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java
new file mode 100644
index 00000000000..56ceae69ff7
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.script.Script;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.util.function.BiFunction;
+
+public class MultiValuesSourceFieldConfig implements Writeable, ToXContentFragment {
+    private String fieldName;
+    private Object missing;
+    private Script script;
+    private DateTimeZone timeZone;
+
+    private static final String NAME = "field_config";
+
+    public static final BiFunction<Boolean, Boolean, ObjectParser<MultiValuesSourceFieldConfig.Builder, Void>> PARSER
+        = (scriptable, timezoneAware) -> {
+
+        ObjectParser<MultiValuesSourceFieldConfig.Builder, Void> parser
+            = new ObjectParser<>(MultiValuesSourceFieldConfig.NAME, MultiValuesSourceFieldConfig.Builder::new);
+
+        parser.declareString(MultiValuesSourceFieldConfig.Builder::setFieldName, ParseField.CommonFields.FIELD);
+        parser.declareField(MultiValuesSourceFieldConfig.Builder::setMissing, XContentParser::objectText,
+            ParseField.CommonFields.MISSING, ObjectParser.ValueType.VALUE);
+
+        if (scriptable) {
+            parser.declareField(MultiValuesSourceFieldConfig.Builder::setScript,
+                (p, context) -> Script.parse(p),
+                Script.SCRIPT_PARSE_FIELD, ObjectParser.ValueType.OBJECT_OR_STRING);
+        }
+
+        if (timezoneAware) {
+            parser.declareField(MultiValuesSourceFieldConfig.Builder::setTimeZone, p -> {
+                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                    return DateTimeZone.forID(p.text());
+                } else {
+                    return DateTimeZone.forOffsetHours(p.intValue());
+                }
+            }, ParseField.CommonFields.TIME_ZONE, ObjectParser.ValueType.LONG);
+        }
+        return parser;
+    };
+
+    private MultiValuesSourceFieldConfig(String fieldName, Object missing, Script script, DateTimeZone timeZone) {
+        this.fieldName = fieldName;
+        this.missing = missing;
+        this.script = script;
+        this.timeZone = timeZone;
+    }
+
+    public MultiValuesSourceFieldConfig(StreamInput in) throws IOException {
+        this.fieldName = in.readString();
+        this.missing = in.readGenericValue();
+        this.script = in.readOptionalWriteable(Script::new);
+        this.timeZone = in.readOptionalTimeZone();
+    }
+
+    public Object getMissing() {
+        return missing;
+    }
+
+    public Script getScript() {
+        return script;
+    }
+
+    public DateTimeZone getTimeZone() {
+        return timeZone;
+    }
+
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(fieldName);
+        out.writeGenericValue(missing);
+        out.writeOptionalWriteable(script);
+        out.writeOptionalTimeZone(timeZone);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (missing != null) {
+            builder.field(ParseField.CommonFields.MISSING.getPreferredName(), missing);
+        }
+        if (script != null) {
+            builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName(), script);
+        }
+        if (fieldName != null) {
+            builder.field(ParseField.CommonFields.FIELD.getPreferredName(), fieldName);
+        }
+        if (timeZone != null) {
+            builder.field(ParseField.CommonFields.TIME_ZONE.getPreferredName(), timeZone);
+        }
+        return builder;
+    }
+
+    public static class Builder {
+        private String fieldName;
+        private Object missing = null;
+        private Script script = null;
+        private DateTimeZone timeZone = null;
+
+        public String getFieldName() {
+            return fieldName;
+        }
+
+        public Builder setFieldName(String fieldName) {
+            this.fieldName = fieldName;
+            return this;
+        }
+
+        public Object getMissing() {
+            return missing;
+        }
+
+        public Builder setMissing(Object missing) {
+            this.missing = missing;
+            return this;
+        }
+
+        public Script getScript() {
+            return script;
+        }
+
+        public Builder setScript(Script script) {
+            this.script = script;
+            return this;
+        }
+
+        public DateTimeZone getTimeZone() {
+            return timeZone;
+        }
+
+        public Builder setTimeZone(DateTimeZone timeZone) {
+            this.timeZone = timeZone;
+            return this;
+        }
+
+        public MultiValuesSourceFieldConfig build() {
+            if (Strings.isNullOrEmpty(fieldName) && script == null) {
+                throw new IllegalArgumentException("[" +  ParseField.CommonFields.FIELD.getPreferredName()
+                    + "] and [" + Script.SCRIPT_PARSE_FIELD.getPreferredName() + "] cannot both be null.  " +
+                    "Please specify one or the other.");
+            }
+
+            if (Strings.isNullOrEmpty(fieldName) == false && script != null) {
+                throw new IllegalArgumentException("[" +  ParseField.CommonFields.FIELD.getPreferredName()
+                    + "] and [" + Script.SCRIPT_PARSE_FIELD.getPreferredName() + "] cannot both be configured.  " +
+                    "Please specify one or the other.");
+            }
+
+            return new MultiValuesSourceFieldConfig(fieldName, missing, script, timeZone);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java
new file mode 100644
index 00000000000..4888495f9d8
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.AbstractObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+public final class MultiValuesSourceParseHelper {
+
+    public static <VS extends ValuesSource, T> void declareCommon(
+        AbstractObjectParser<? extends MultiValuesSourceAggregationBuilder<VS, ?>, T> objectParser, boolean formattable,
+        ValueType targetValueType) {
+
+        objectParser.declareField(MultiValuesSourceAggregationBuilder::valueType, p -> {
+            ValueType valueType = ValueType.resolveForScript(p.text());
+            if (targetValueType != null && valueType.isNotA(targetValueType)) {
+                throw new ParsingException(p.getTokenLocation(),
+                    "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type ["
+                        + valueType + "]. It can only work on value of type ["
+                        + targetValueType + "]");
+            }
+            return valueType;
+        }, ValueType.VALUE_TYPE, ObjectParser.ValueType.STRING);
+
+        if (formattable) {
+            objectParser.declareField(MultiValuesSourceAggregationBuilder::format, XContentParser::text,
+                ParseField.CommonFields.FORMAT, ObjectParser.ValueType.STRING);
+        }
+    }
+
+    public static <VS extends ValuesSource, T> void declareField(String fieldName,
+        AbstractObjectParser<? extends MultiValuesSourceAggregationBuilder<VS, ?>, T> objectParser,
+        boolean scriptable, boolean timezoneAware) {
+
+        objectParser.declareField((o, fieldConfig) -> o.field(fieldName, fieldConfig.build()),
+            (p, c) -> MultiValuesSourceFieldConfig.PARSER.apply(scriptable, timezoneAware).parse(p, null),
+            new ParseField(fieldName), ObjectParser.ValueType.OBJECT);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
index 318540e3e58..7f6e76a6611 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.aggregations.support;
 
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -95,6 +96,8 @@ public enum ValueType implements Writeable {
     private final byte id;
     private String preferredName;
 
+    public static final ParseField VALUE_TYPE = new ParseField("value_type", "valueType");
+
     ValueType(byte id, String description, String preferredName, ValuesSourceType valuesSourceType,
             Class<? extends IndexFieldData> fieldDataType, DocValueFormat defaultFormat) {
         this.id = id;
@@ -112,7 +115,7 @@ public enum ValueType implements Writeable {
     public String getPreferredName() {
         return preferredName;
     }
-    
+
     public ValuesSourceType getValuesSourceType() {
         return valuesSourceType;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java
index 365233122c4..fc0a2f3a9fe 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java
@@ -28,7 +28,6 @@ import org.elasticsearch.script.Script;
 import org.joda.time.DateTimeZone;
 
 public final class ValuesSourceParserHelper {
-    static final ParseField TIME_ZONE = new ParseField("time_zone");
 
     private ValuesSourceParserHelper() {} // utility class, no instantiation
 
@@ -62,10 +61,10 @@ public final class ValuesSourceParserHelper {
 
 
         objectParser.declareField(ValuesSourceAggregationBuilder::field, XContentParser::text,
-                new ParseField("field"), ObjectParser.ValueType.STRING);
+            ParseField.CommonFields.FIELD, ObjectParser.ValueType.STRING);
 
         objectParser.declareField(ValuesSourceAggregationBuilder::missing, XContentParser::objectText,
-                new ParseField("missing"), ObjectParser.ValueType.VALUE);
+            ParseField.CommonFields.MISSING, ObjectParser.ValueType.VALUE);
 
         objectParser.declareField(ValuesSourceAggregationBuilder::valueType, p -> {
             ValueType valueType = ValueType.resolveForScript(p.text());
@@ -76,11 +75,11 @@ public final class ValuesSourceParserHelper {
                                 + targetValueType + "]");
             }
             return valueType;
-        }, new ParseField("value_type", "valueType"), ObjectParser.ValueType.STRING);
+        }, ValueType.VALUE_TYPE, ObjectParser.ValueType.STRING);
 
         if (formattable) {
             objectParser.declareField(ValuesSourceAggregationBuilder::format, XContentParser::text,
-                    new ParseField("format"), ObjectParser.ValueType.STRING);
+                ParseField.CommonFields.FORMAT, ObjectParser.ValueType.STRING);
         }
 
         if (scriptable) {
@@ -96,7 +95,7 @@ public final class ValuesSourceParserHelper {
                 } else {
                     return DateTimeZone.forOffsetHours(p.intValue());
                 }
-            }, TIME_ZONE, ObjectParser.ValueType.LONG);
+            }, ParseField.CommonFields.TIME_ZONE, ObjectParser.ValueType.LONG);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java
index a6b252d6903..387e807ba86 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceType.java
@@ -19,9 +19,37 @@
 
 package org.elasticsearch.search.aggregations.support;
 
-public enum ValuesSourceType  {
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Locale;
+
+public enum ValuesSourceType implements Writeable {
     ANY,
     NUMERIC,
     BYTES,
     GEOPOINT;
+
+    public static final ParseField VALUE_SOURCE_TYPE = new ParseField("value_source_type");
+
+    public static ValuesSourceType fromString(String name) {
+        return valueOf(name.trim().toUpperCase(Locale.ROOT));
+    }
+
+    public static ValuesSourceType fromStream(StreamInput in) throws IOException {
+        return in.readEnum(ValuesSourceType.class);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        ValuesSourceType state = this;
+        out.writeEnum(state);
+    }
+
+    public String value() {
+        return name().toLowerCase(Locale.ROOT);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregatorTests.java
new file mode 100644
index 00000000000..70b1b651723
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregatorTests.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.weighted_avg;
+
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.function.Consumer;
+
+import static java.util.Collections.singleton;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class WeightedAvgAggregatorTests extends AggregatorTestCase {
+
+    public void testNoDocs() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+            // Intentionally not writing any docs
+        }, avg -> {
+            assertEquals(Double.NaN, avg.getValue(), 0);
+        });
+    }
+
+    public void testNoMatchingField() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 3)));
+        }, avg -> {
+            assertEquals(Double.NaN, avg.getValue(), 0);
+        });
+    }
+
+    public void testSomeMatchesSortedNumericDocValuesNoWeight() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 7),
+                new SortedNumericDocValuesField("weight_field", 1)));
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 2),
+                new SortedNumericDocValuesField("weight_field", 1)));
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 3),
+                new SortedNumericDocValuesField("weight_field", 1)));
+        }, avg -> {
+            assertEquals(4, avg.getValue(), 0);
+        });
+    }
+
+    public void testSomeMatchesSortedNumericDocValuesWeights() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 7),
+                new SortedNumericDocValuesField("weight_field", 2)));
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 2),
+                new SortedNumericDocValuesField("weight_field", 3)));
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 3),
+                new SortedNumericDocValuesField("weight_field", 3)));
+
+        }, avg -> {
+            // (7*2 + 2*3 + 3*3) / (2+3+3) == 3.625
+            assertEquals(3.625, avg.getValue(), 0);
+        });
+    }
+
+    public void testSomeMatchesNumericDocValues() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(new DocValuesFieldExistsQuery("value_field"), aggregationBuilder, iw -> {
+            iw.addDocument(Arrays.asList(new NumericDocValuesField("value_field", 7),
+                new SortedNumericDocValuesField("weight_field", 1)));
+            iw.addDocument(Arrays.asList(new NumericDocValuesField("value_field", 2),
+                new SortedNumericDocValuesField("weight_field", 1)));
+            iw.addDocument(Arrays.asList(new NumericDocValuesField("value_field", 3),
+                new SortedNumericDocValuesField("weight_field", 1)));
+        }, avg -> {
+            assertEquals(4, avg.getValue(), 0);
+        });
+    }
+
+    public void testQueryFiltering() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(IntPoint.newRangeQuery("value_field", 0, 3), aggregationBuilder, iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("value_field", 7), new SortedNumericDocValuesField("value_field", 7),
+                new SortedNumericDocValuesField("weight_field", 1)));
+            iw.addDocument(Arrays.asList(new IntPoint("value_field", 1), new SortedNumericDocValuesField("value_field", 2),
+                new SortedNumericDocValuesField("weight_field", 1)));
+            iw.addDocument(Arrays.asList(new IntPoint("value_field", 3), new SortedNumericDocValuesField("value_field", 3),
+                new SortedNumericDocValuesField("weight_field", 1)));
+        }, avg -> {
+            assertEquals(2.5, avg.getValue(), 0);
+        });
+    }
+
+    public void testQueryFilteringWeights() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(IntPoint.newRangeQuery("filter_field", 0, 3), aggregationBuilder, iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("filter_field", 7), new SortedNumericDocValuesField("value_field", 7),
+                new SortedNumericDocValuesField("weight_field", 2)));
+            iw.addDocument(Arrays.asList(new IntPoint("filter_field", 2), new SortedNumericDocValuesField("value_field", 2),
+                new SortedNumericDocValuesField("weight_field", 3)));
+            iw.addDocument(Arrays.asList(new IntPoint("filter_field", 3), new SortedNumericDocValuesField("value_field", 3),
+                new SortedNumericDocValuesField("weight_field", 4)));
+        }, avg -> {
+            double value = (2.0*3.0 + 3.0*4.0) / (3.0+4.0);
+            assertEquals(value, avg.getValue(), 0);
+        });
+    }
+
+    public void testQueryFiltersAll() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(IntPoint.newRangeQuery("value_field", -1, 0), aggregationBuilder, iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("value_field", 7), new SortedNumericDocValuesField("value_field", 7)));
+            iw.addDocument(Arrays.asList(new IntPoint("value_field", 1), new SortedNumericDocValuesField("value_field", 2)));
+            iw.addDocument(Arrays.asList(new IntPoint("value_field", 3), new SortedNumericDocValuesField("value_field", 7)));
+        }, avg -> {
+            assertEquals(Double.NaN, avg.getValue(), 0);
+        });
+    }
+
+    public void testQueryFiltersAllWeights() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(IntPoint.newRangeQuery("value_field", -1, 0), aggregationBuilder, iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("filter_field", 7), new SortedNumericDocValuesField("value_field", 7),
+                new SortedNumericDocValuesField("weight_field", 2)));
+            iw.addDocument(Arrays.asList(new IntPoint("filter_field", 2), new SortedNumericDocValuesField("value_field", 2),
+                new SortedNumericDocValuesField("weight_field", 3)));
+            iw.addDocument(Arrays.asList(new IntPoint("filter_field", 3), new SortedNumericDocValuesField("value_field", 3),
+                new SortedNumericDocValuesField("weight_field", 4)));
+        }, avg -> {
+            assertEquals(Double.NaN, avg.getValue(), 0);
+        });
+    }
+
+    public void testValueSetMissing() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder()
+            .setFieldName("value_field")
+            .setMissing(2)
+            .build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+            iw.addDocument(Collections.singletonList(new SortedNumericDocValuesField("weight_field", 2)));
+            iw.addDocument(Collections.singletonList(new SortedNumericDocValuesField("weight_field", 3)));
+            iw.addDocument(Collections.singletonList(new SortedNumericDocValuesField("weight_field", 4)));
+        }, avg -> {
+            double value = (2.0*2.0 + 2.0*3.0 + 2.0*4.0) / (2.0+3.0+4.0);
+            assertEquals(value, avg.getValue(), 0);
+        });
+    }
+
+    public void testWeightSetMissing() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig =  new MultiValuesSourceFieldConfig.Builder()
+            .setFieldName("weight_field")
+            .setMissing(2)
+            .build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+            iw.addDocument(Collections.singletonList(new SortedNumericDocValuesField("value_field", 2)));
+            iw.addDocument(Collections.singletonList(new SortedNumericDocValuesField("value_field", 3)));
+            iw.addDocument(Collections.singletonList(new SortedNumericDocValuesField("value_field", 4)));
+        }, avg -> {
+            double value = (2.0*2.0 + 3.0*2.0 + 4.0*2.0) / (2.0+2.0+2.0);
+            assertEquals(value, avg.getValue(), 0);
+        });
+    }
+
+    public void testWeightSetTimezone() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder()
+            .setFieldName("weight_field")
+            .setTimeZone(DateTimeZone.UTC)
+            .build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+                iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 2),
+                    new SortedNumericDocValuesField("weight_field", 1)));
+                iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 3),
+                    new SortedNumericDocValuesField("weight_field", 1)));
+                iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 4),
+                    new SortedNumericDocValuesField("weight_field", 1)));
+            }, avg -> {
+               fail("Should not have executed test case");
+            }));
+        assertThat(e.getMessage(), equalTo("Field [weight_field] of type [long] does not support custom time zones"));
+    }
+
+    public void testValueSetTimezone() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder()
+            .setFieldName("value_field")
+            .setTimeZone(DateTimeZone.UTC)
+            .build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+                iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 2),
+                    new SortedNumericDocValuesField("weight_field", 1)));
+                iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 3),
+                    new SortedNumericDocValuesField("weight_field", 1)));
+                iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 4),
+                    new SortedNumericDocValuesField("weight_field", 1)));
+            }, avg -> {
+                fail("Should not have executed test case");
+            }));
+        assertThat(e.getMessage(), equalTo("Field [value_field] of type [long] does not support custom time zones"));
+    }
+
+    public void testMultiValues() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder()
+            .setFieldName("value_field")
+            .build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+
+        testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 2),
+                new SortedNumericDocValuesField("value_field", 3), new SortedNumericDocValuesField("weight_field", 1)));
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 3),
+                new SortedNumericDocValuesField("value_field", 4), new SortedNumericDocValuesField("weight_field", 1)));
+            iw.addDocument(Arrays.asList(new SortedNumericDocValuesField("value_field", 4),
+                new SortedNumericDocValuesField("value_field", 5), new SortedNumericDocValuesField("weight_field", 1)));
+        }, avg -> {
+            double value = (((2.0+3.0)/2.0) + ((3.0+4.0)/2.0) + ((4.0+5.0)/2.0)) / (1.0+1.0+1.0);
+            assertEquals(value, avg.getValue(), 0);
+        });
+    }
+
+    public void testMultiWeight() throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder()
+            .setFieldName("weight_field")
+            .build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+
+        AggregationExecutionException e = expectThrows(AggregationExecutionException.class,
+            () -> testCase(new MatchAllDocsQuery(), aggregationBuilder, iw -> {
+                iw.addDocument(Arrays.asList(
+                    new SortedNumericDocValuesField("value_field", 2),
+                    new SortedNumericDocValuesField("weight_field", 2), new SortedNumericDocValuesField("weight_field", 3)));
+                iw.addDocument(Arrays.asList(
+                    new SortedNumericDocValuesField("value_field", 3),
+                    new SortedNumericDocValuesField("weight_field", 3), new SortedNumericDocValuesField("weight_field", 4)));
+                iw.addDocument(Arrays.asList(
+                    new SortedNumericDocValuesField("value_field", 4),
+                    new SortedNumericDocValuesField("weight_field", 4), new SortedNumericDocValuesField("weight_field", 5)));
+            }, avg -> {
+                fail("Should have thrown exception");
+            }));
+        assertThat(e.getMessage(), containsString("Encountered more than one weight for a single document. " +
+            "Use a script to combine multiple weights-per-doc into a single value."));
+    }
+
+
+    public void testSummationAccuracy() throws IOException {
+        // Summing up a normal array and expect an accurate value
+        double[] values = new double[]{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.9, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7};
+        verifyAvgOfDoubles(values, 0.9, 0d);
+
+        // Summing up an array which contains NaN and infinities and expect a result same as naive summation
+        int n = randomIntBetween(5, 10);
+        values = new double[n];
+        double sum = 0;
+        for (int i = 0; i < n; i++) {
+            values[i] = frequently()
+                ? randomFrom(Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY)
+                : randomDoubleBetween(Double.MIN_VALUE, Double.MAX_VALUE, true);
+            sum += values[i];
+        }
+        verifyAvgOfDoubles(values, sum / n, 1e-10);
+
+        // Summing up some big double values and expect infinity result
+        n = randomIntBetween(5, 10);
+        double[] largeValues = new double[n];
+        for (int i = 0; i < n; i++) {
+            largeValues[i] = Double.MAX_VALUE;
+        }
+        verifyAvgOfDoubles(largeValues, Double.POSITIVE_INFINITY, 0d);
+
+        for (int i = 0; i < n; i++) {
+            largeValues[i] = -Double.MAX_VALUE;
+        }
+        verifyAvgOfDoubles(largeValues, Double.NEGATIVE_INFINITY, 0d);
+    }
+
+    private void verifyAvgOfDoubles(double[] values, double expected, double delta) throws IOException {
+        MultiValuesSourceFieldConfig valueConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("value_field").build();
+        MultiValuesSourceFieldConfig weightConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName("weight_field").build();
+        WeightedAvgAggregationBuilder aggregationBuilder = new WeightedAvgAggregationBuilder("_name")
+            .value(valueConfig)
+            .weight(weightConfig);
+        testCase(new MatchAllDocsQuery(), aggregationBuilder,
+            iw -> {
+                for (double value : values) {
+                    iw.addDocument(Arrays.asList(new NumericDocValuesField("value_field", NumericUtils.doubleToSortableLong(value)),
+                        new SortedNumericDocValuesField("weight_field", NumericUtils.doubleToSortableLong(1.0))));
+                }
+            },
+            avg -> assertEquals(expected, avg.getValue(), delta),
+            NumberFieldMapper.NumberType.DOUBLE
+        );
+    }
+
+    private void testCase(Query query, WeightedAvgAggregationBuilder aggregationBuilder,
+                          CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
+                          Consumer<InternalWeightedAvg> verify) throws IOException {
+        testCase(query, aggregationBuilder, buildIndex, verify, NumberFieldMapper.NumberType.LONG);
+    }
+
+    private void testCase(Query query, WeightedAvgAggregationBuilder aggregationBuilder,
+                          CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
+                          Consumer<InternalWeightedAvg> verify,
+                          NumberFieldMapper.NumberType fieldNumberType) throws IOException {
+
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        buildIndex.accept(indexWriter);
+        indexWriter.close();
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        try {
+            MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(fieldNumberType);
+            fieldType.setName("value_field");
+            fieldType.setHasDocValues(true);
+
+            MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(fieldNumberType);
+            fieldType2.setName("weight_field");
+            fieldType2.setHasDocValues(true);
+
+            WeightedAvgAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType, fieldType2);
+            aggregator.preCollection();
+            indexSearcher.search(query, aggregator);
+            aggregator.postCollection();
+            verify.accept((InternalWeightedAvg) aggregator.buildAggregation(0L));
+        } finally {
+            indexReader.close();
+            directory.close();
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java
new file mode 100644
index 00000000000..ac1c07a4049
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.script.Script;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MultiValuesSourceFieldConfigTests extends ESTestCase {
+    public void testMissingFieldScript() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new MultiValuesSourceFieldConfig.Builder().build());
+        assertThat(e.getMessage(), equalTo("[field] and [script] cannot both be null.  Please specify one or the other."));
+    }
+
+    public void testBothFieldScript() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new MultiValuesSourceFieldConfig.Builder().setFieldName("foo").setScript(new Script("foo")).build());
+        assertThat(e.getMessage(), equalTo("[field] and [script] cannot both be configured.  Please specify one or the other."));
+    }
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java
index dc2fac776c6..538babf4fbc 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupRequestTranslator.java
@@ -208,7 +208,7 @@ public class RollupRequestTranslator {
     private static List<AggregationBuilder> translateDateHistogram(DateHistogramAggregationBuilder source,
                                                                    List<QueryBuilder> filterConditions,
                                                                    NamedWriteableRegistry registry) {
-        
+
         return translateVSAggBuilder(source, filterConditions, registry, () -> {
             DateHistogramAggregationBuilder rolledDateHisto
                     = new DateHistogramAggregationBuilder(source.getName());

From e2b665c2e6caa5cc78484247f591f437ab99fac5 Mon Sep 17 00:00:00 2001
From: Russ Cam <russ.cam@elastic.co>
Date: Tue, 24 Jul 2018 09:21:43 +1000
Subject: [PATCH 147/260] Consistent encoder names (#29492)

This commit updates encoder names to be consistent within documentation
and align with snake casing convention.
---
 docs/plugins/analysis-phonetic.asciidoc                   | 6 +++---
 .../org/elasticsearch/index/analysis/phonetic-1.yml       | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc
index 34f4a933f9f..a75c21fdac6 100644
--- a/docs/plugins/analysis-phonetic.asciidoc
+++ b/docs/plugins/analysis-phonetic.asciidoc
@@ -17,15 +17,15 @@ The `phonetic` token filter takes the following settings:
 `encoder`::
 
     Which phonetic encoder to use.  Accepts `metaphone` (default),
-    `doublemetaphone`, `soundex`, `refinedsoundex`, `caverphone1`,
+    `double_metaphone`, `soundex`, `refined_soundex`, `caverphone1`,
     `caverphone2`, `cologne`, `nysiis`, `koelnerphonetik`, `haasephonetik`,
-    `beidermorse`, `daitch_mokotoff`.
+    `beider_morse`, `daitch_mokotoff`.
 
 `replace`::
 
     Whether or not the original token should be replaced by the phonetic
     token. Accepts `true` (default) and `false`.  Not supported by
-    `beidermorse` encoding.
+    `beider_morse` encoding.
 
 [source,js]
 --------------------------------------------------
diff --git a/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml b/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml
index 1909c7ee063..14cdbbe2440 100644
--- a/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml
+++ b/plugins/analysis-phonetic/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml
@@ -3,7 +3,7 @@ index:
     filter:
       doublemetaphonefilter:
         type: phonetic
-        encoder: doublemetaphone
+        encoder: double_metaphone
       metaphonefilter:
         type: phonetic
         encoder: metaphone
@@ -12,16 +12,16 @@ index:
         encoder: soundex
       refinedsoundexfilter:
         type: phonetic
-        encoder: refinedsoundex
+        encoder: refined_soundex
       caverphonefilter:
         type: phonetic
         encoder: caverphone
       beidermorsefilter:
         type: phonetic
-        encoder: beidermorse
+        encoder: beider_morse
       beidermorsefilterfrench:
         type: phonetic
-        encoder: beidermorse
+        encoder: beider_morse
         languageset : [ "french" ]
       koelnerphonetikfilter:
         type: phonetic

From 7c43da280b4689951a7b30e50e0040e3ad41f79a Mon Sep 17 00:00:00 2001
From: Zachary Tong <zach@elastic.co>
Date: Mon, 23 Jul 2018 19:50:26 -0400
Subject: [PATCH 148/260] [TEST] add version skip to weighted_avg tests

Added in 6.4, so we don't want this running on pre-6.4 BWC tests
---
 .../rest-api-spec/test/search.aggregation/260_weighted_avg.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml
index bccc9619233..6f8ea9565e3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/260_weighted_avg.yml
@@ -1,4 +1,7 @@
 setup:
+  - skip:
+      version: " - 6.3.99"
+      reason: weighted_avg is only available as of 6.4.0
   - do:
       indices.create:
           index: test_1

From d07b4ecfa358968836aadfce025fcf291dea0ac6 Mon Sep 17 00:00:00 2001
From: Andy Bristol <andy.bristol@elastic.co>
Date: Mon, 23 Jul 2018 17:20:01 -0700
Subject: [PATCH 149/260] awaitsfix testRandomClusterStateUpdates

For #32308
---
 .../cluster/IndicesClusterStateServiceRandomUpdatesTests.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 3a5302bcec2..5611421594a 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -97,6 +97,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
         terminate(threadPool);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32308")
     public void testRandomClusterStateUpdates() {
         // we have an IndicesClusterStateService per node in the cluster
         final Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap = new HashMap<>();

From 177750719db0b274c98ab68dbc719613a8fe98f0 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Tue, 24 Jul 2018 10:21:34 +0100
Subject: [PATCH 150/260] [DOCS] Rollup Caps API incorrectly mentions GET Jobs
 API (#32280)

---
 x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc
index 270ad005144..f770adf1f0d 100644
--- a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc
+++ b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc
@@ -33,7 +33,7 @@ live?
 
 ==== Request Body
 
-There is no request body for the Get Jobs API.
+There is no request body for the Get Rollup Caps API.
 
 ==== Authorization
 
@@ -179,4 +179,4 @@ GET _xpack/rollup/data/sensor-1
 Why is this?  The original rollup job was configured against a specific index pattern (`sensor-*`) not a concrete index
 (`sensor-1`).  So while the index belongs to the pattern, the rollup job is only valid across the entirety of the pattern
 not just one of it's containing indices.  So for that reason, the Rollup Capabilities API only returns information based
-on the originally configured index name or pattern.
\ No newline at end of file
+on the originally configured index name or pattern.

From a2dbd83db1f5b3ad32c315b48b8229a6269b0438 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ikakavas@protonmail.com>
Date: Tue, 24 Jul 2018 12:48:14 +0300
Subject: [PATCH 151/260] Allow Integ Tests to run in a FIPS-140 JVM (#31989)

* Complete changes for running IT in a fips JVM

- Mute :x-pack:qa:sql:security:ssl:integTest as it
  cannot run in FIPS 140 JVM until the SQL CLI supports key/cert.
- Set default JVM keystore/truststore password in top level build
  script for all integTest tasks in a FIPS 140 JVM
- Changed top level x-pack build script to use keys and certificates
  for trust/key material when spinning up clusters for IT
---
 .../elasticsearch/gradle/BuildPlugin.groovy   |  11 +-
 plugins/discovery-gce/build.gradle            |  30 --
 .../action/admin/ReloadSecureSettingsIT.java  |   9 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   4 +
 x-pack/plugin/build.gradle                    |  47 ++-
 .../rest-api-spec/test/ssl/10_basic.yml       |   5 +-
 x-pack/qa/full-cluster-restart/build.gradle   |   4 +-
 .../ml-native-multi-node-tests/build.gradle   |  46 +--
 .../MlNativeAutodetectIntegTestCase.java      |  11 +-
 x-pack/qa/rolling-upgrade/build.gradle        |   2 +-
 x-pack/qa/smoke-test-plugins-ssl/build.gradle | 320 ++----------------
 ...keTestPluginsSslClientYamlTestSuiteIT.java |   4 +-
 .../qa/smoke-test-plugins-ssl/testclient.crt  |  23 ++
 .../qa/smoke-test-plugins-ssl/testclient.jks  | Bin 0 -> 3358 bytes
 .../qa/smoke-test-plugins-ssl/testclient.pem  |  30 ++
 x-pack/qa/smoke-test-plugins-ssl/testnode.crt |  23 ++
 x-pack/qa/smoke-test-plugins-ssl/testnode.jks | Bin 0 -> 9360 bytes
 x-pack/qa/smoke-test-plugins-ssl/testnode.pem |  30 ++
 x-pack/qa/sql/security/ssl/build.gradle       |  26 +-
 .../qa/sql/jdbc/JdbcIntegrationTestCase.java  |   2 +-
 20 files changed, 224 insertions(+), 403 deletions(-)
 create mode 100644 x-pack/qa/smoke-test-plugins-ssl/testclient.crt
 create mode 100644 x-pack/qa/smoke-test-plugins-ssl/testclient.jks
 create mode 100644 x-pack/qa/smoke-test-plugins-ssl/testclient.pem
 create mode 100644 x-pack/qa/smoke-test-plugins-ssl/testnode.crt
 create mode 100644 x-pack/qa/smoke-test-plugins-ssl/testnode.jks
 create mode 100644 x-pack/qa/smoke-test-plugins-ssl/testnode.pem

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 5da5912dabe..85216cf7336 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -131,6 +131,9 @@ class BuildPlugin implements Plugin<Project> {
                 runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome))
             }
 
+            String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));'
+            boolean inFipsJvm = Boolean.parseBoolean(runJavascript(project, runtimeJavaHome, inFipsJvmScript))
+
             // Build debugging info
             println '======================================='
             println 'Elasticsearch Build Hamster says Hello!'
@@ -202,6 +205,7 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.buildChecksDone = true
             project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
             project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
+            project.rootProject.ext.inFipsJvm = inFipsJvm
         }
 
         project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
@@ -213,6 +217,7 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
         project.ext.javaVersions = project.rootProject.ext.javaVersions
+        project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
     }
 
     private static String findCompilerJavaHome() {
@@ -770,7 +775,11 @@ class BuildPlugin implements Plugin<Project> {
                     systemProperty property.getKey(), property.getValue()
                 }
             }
-
+            // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
+            if (project.inFipsJvm) {
+                systemProperty 'javax.net.ssl.trustStorePassword', 'password'
+                systemProperty 'javax.net.ssl.keyStorePassword', 'password'
+            }
             boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))
             enableSystemAssertions assertionsEnabled
             enableAssertions assertionsEnabled
diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle
index 82de9ba031b..fa8005dfa47 100644
--- a/plugins/discovery-gce/build.gradle
+++ b/plugins/discovery-gce/build.gradle
@@ -22,36 +22,6 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
 }
 
-
-// needed to be consistent with ssl host checking
-String host = InetAddress.getLoopbackAddress().getHostAddress();
-
-// location of keystore and files to generate it
-File keystore = new File(project.buildDir, 'keystore/test-node.jks')
-
-// generate the keystore
-task createKey(type: LoggedExec) {
-  doFirst {
-    project.delete(keystore.parentFile)
-    keystore.parentFile.mkdirs()
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-node',
-          '-keystore', keystore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=' + host,
-          '-keypass', 'keypass',
-          '-storepass', 'keypass'
-}
-
-// add keystore to test classpath: it expects it there
-sourceSets.test.resources.srcDir(keystore.parentFile)
-processTestResources.dependsOn(createKey)
-
 dependencyLicenses {
   mapping from: /google-.*/, to: 'google'
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
index 2061349e330..c8503603f66 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
@@ -205,7 +205,14 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
                             assertThat(nodesMap.size(), equalTo(cluster().size()));
                             for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) {
                                 assertThat(nodeResponse.reloadException(), notNullValue());
-                                assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
+                                // Running in a JVM with a BouncyCastle FIPS Security Provider, decrypting the Keystore with the wrong
+                                // password returns a SecurityException if the DataInputStream can't be fully consumed
+                                if (inFipsJvm()) {
+                                    assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class));
+                                } else {
+                                    assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
+                                }
+
                             }
                         } catch (final AssertionError e) {
                             reloadSettingsError.set(e);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 275bca4d28d..3a479aad897 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -176,6 +176,7 @@ import java.net.InetSocketAddress;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -2364,4 +2365,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         });
     }
 
+    public static boolean inFipsJvm() {
+        return Security.getProviders()[0].getName().toLowerCase(Locale.ROOT).contains("fips");
+    }
 }
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index ca529496bf1..b9cd464241f 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -104,39 +104,28 @@ integTestRunner {
   systemProperty 'tests.rest.blacklist', blacklist.join(',')
 }
 
-// location of generated keystores and certificates
+// location for keys and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
+File nodeKey = file("$keystoreDir/testnode.pem")
+File nodeCert = file("$keystoreDir/testnode.crt")
 
-// Generate the node's keystore
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
-task createNodeKeyStore(type: LoggedExec) {
-  doFirst {
-    if (nodeKeystore.parentFile.exists() == false) {
-      nodeKeystore.parentFile.mkdirs()
+// Add key and certs to test classpath: it expects them there
+// User cert and key PEM files instead of a JKS Keystore for the cluster's trust material so that
+// it can run in a FIPS 140 JVM
+// TODO: Remove all existing uses of cross project file references when the new approach for referencing static files is available
+// https://github.com/elastic/elasticsearch/pull/32201
+task copyKeyCerts(type: Copy) {
+  from(project(':x-pack:plugin:core').file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/')) {
+    include 'testnode.crt', 'testnode.pem'
     }
-    if (nodeKeystore.exists()) {
-      delete nodeKeystore
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-node',
-          '-keystore', nodeKeystore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=smoke-test-plugins-ssl',
-          '-keypass', 'keypass',
-          '-storepass', 'keypass'
+  into keystoreDir
 }
-
 // Add keystores to test classpath: it expects it there
 sourceSets.test.resources.srcDir(keystoreDir)
-processTestResources.dependsOn(createNodeKeyStore)
+processTestResources.dependsOn(copyKeyCerts)
 
 integTestCluster {
-  dependsOn createNodeKeyStore
+  dependsOn copyKeyCerts
   setting 'xpack.ml.enabled', 'true'
   setting 'xpack.security.enabled', 'true'
   setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
@@ -145,17 +134,19 @@ integTestCluster {
   setting 'xpack.monitoring.exporters._local.enabled', 'false'
   setting 'xpack.security.authc.token.enabled', 'true'
   setting 'xpack.security.transport.ssl.enabled', 'true'
-  setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
+  setting 'xpack.security.transport.ssl.key', nodeKey.name
+  setting 'xpack.security.transport.ssl.certificate', nodeCert.name
   setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
   setting 'xpack.security.audit.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
   keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-  keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
+  keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
   distribution = 'zip' // this is important since we use the reindex module in ML
 
   setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
 
-  extraConfigFile nodeKeystore.name, nodeKeystore
+  extraConfigFile nodeKey.name, nodeKey
+  extraConfigFile nodeCert.name, nodeCert
 
   waitCondition = { NodeInfo node, AntBuilder ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ssl/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ssl/10_basic.yml
index 7a87ef511e5..227d341b26d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ssl/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ssl/10_basic.yml
@@ -4,7 +4,6 @@
       xpack.ssl.certificates: {}
 
   - length: { $body: 1 }
-  - match: { $body.0.path: "test-node.jks" }
-  - match: { $body.0.format: "jks" }
-  - match: { $body.0.alias: "test-node" }
+  - match: { $body.0.path: "testnode.crt" }
+  - match: { $body.0.format: "PEM" }
   - match: { $body.0.has_private_key: true }
diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle
index 78ac1436fd8..5a6f0c9cc83 100644
--- a/x-pack/qa/full-cluster-restart/build.gradle
+++ b/x-pack/qa/full-cluster-restart/build.gradle
@@ -125,8 +125,8 @@ subprojects {
 
   String output = "${buildDir}/generated-resources/${project.name}"
   task copyTestNodeKeystore(type: Copy) {
-    from project(xpackModule('core'))
-            .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
+    from project(':x-pack:plugin:core')
+      .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
     into outputDir
   }
 
diff --git a/x-pack/qa/ml-native-multi-node-tests/build.gradle b/x-pack/qa/ml-native-multi-node-tests/build.gradle
index 286d4daee8a..b8627db3068 100644
--- a/x-pack/qa/ml-native-multi-node-tests/build.gradle
+++ b/x-pack/qa/ml-native-multi-node-tests/build.gradle
@@ -18,59 +18,45 @@ integTestRunner {
     systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
-// location of generated keystores and certificates
+// location for keys and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
-
-// Generate the node's keystore
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
-task createNodeKeyStore(type: LoggedExec) {
-    doFirst {
-        if (nodeKeystore.parentFile.exists() == false) {
-            nodeKeystore.parentFile.mkdirs()
-        }
-        if (nodeKeystore.exists()) {
-            delete nodeKeystore
-        }
+File nodeKey = file("$keystoreDir/testnode.pem")
+File nodeCert = file("$keystoreDir/testnode.crt")
+// Add key and certs to test classpath: it expects it there
+task copyKeyCerts(type: Copy) {
+    from(project(':x-pack:plugin:core').file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/')) {
+        include 'testnode.crt', 'testnode.pem'
     }
-    executable = new File(project.runtimeJavaHome, 'bin/keytool')
-    standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-    args '-genkey',
-            '-alias', 'test-node',
-            '-keystore', nodeKeystore,
-            '-keyalg', 'RSA',
-            '-keysize', '2048',
-            '-validity', '712',
-            '-dname', 'CN=smoke-test-plugins-ssl',
-            '-keypass', 'keypass',
-            '-storepass', 'keypass'
+    into keystoreDir
 }
-
-// Add keystores to test classpath: it expects it there
+// Add keys and cets to test classpath: it expects it there
 sourceSets.test.resources.srcDir(keystoreDir)
-processTestResources.dependsOn(createNodeKeyStore)
+processTestResources.dependsOn(copyKeyCerts)
 
 integTestCluster {
-    dependsOn createNodeKeyStore
+    dependsOn copyKeyCerts
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.ml.enabled', 'true'
     setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.security.authc.token.enabled', 'true'
     setting 'xpack.security.transport.ssl.enabled', 'true'
-    setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
+    setting 'xpack.security.transport.ssl.key', nodeKey.name
+    setting 'xpack.security.transport.ssl.certificate', nodeCert.name
     setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
     setting 'xpack.security.audit.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
 
     keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-    keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
+    keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
 
     numNodes = 3
 
     setupCommand 'setupDummyUser',
             'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
 
-    extraConfigFile nodeKeystore.name, nodeKeystore
+    extraConfigFile nodeKey.name, nodeKey
+    extraConfigFile nodeCert.name, nodeCert
 
     waitCondition = { node, ant ->
         File tmpFile = new File(node.cwd, 'wait.success')
diff --git a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
index c9956e6092f..74a026ed5ad 100644
--- a/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
+++ b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
@@ -124,9 +124,11 @@ abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase {
 
     @Override
     protected Settings externalClusterClientSettings() {
-        Path keyStore;
+        Path key;
+        Path certificate;
         try {
-            keyStore = PathUtils.get(getClass().getResource("/test-node.jks").toURI());
+            key = PathUtils.get(getClass().getResource("/testnode.pem").toURI());
+            certificate = PathUtils.get(getClass().getResource("/testnode.crt").toURI());
         } catch (URISyntaxException e) {
             throw new IllegalStateException("error trying to get keystore path", e);
         }
@@ -135,8 +137,9 @@ abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase {
         builder.put(SecurityField.USER_SETTING.getKey(), "x_pack_rest_user:" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true);
         builder.put("xpack.security.transport.ssl.enabled", true);
-        builder.put("xpack.security.transport.ssl.keystore.path", keyStore.toAbsolutePath().toString());
-        builder.put("xpack.security.transport.ssl.keystore.password", "keypass");
+        builder.put("xpack.security.transport.ssl.key", key.toAbsolutePath().toString());
+        builder.put("xpack.security.transport.ssl.certificate", certificate.toAbsolutePath().toString());
+        builder.put("xpack.security.transport.ssl.key_passphrase", "testnode");
         builder.put("xpack.security.transport.ssl.verification_mode", "certificate");
         return builder.build();
     }
diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle
index 351f33b9412..f60cbe4b34c 100644
--- a/x-pack/qa/rolling-upgrade/build.gradle
+++ b/x-pack/qa/rolling-upgrade/build.gradle
@@ -107,7 +107,7 @@ subprojects {
 
   String output = "${buildDir}/generated-resources/${project.name}"
   task copyTestNodeKeystore(type: Copy) {
-    from project(xpackModule('core'))
+    from project(':x-pack:plugin:core')
             .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
     into outputDir
   }
diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
index 595c562af37..672bc3a19e8 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle
+++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
@@ -4,7 +4,7 @@ import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.test.NodeInfo
 
 import javax.net.ssl.HttpsURLConnection
-import javax.net.ssl.KeyManagerFactory
+import javax.net.ssl.KeyManager
 import javax.net.ssl.SSLContext
 import javax.net.ssl.TrustManagerFactory
 import java.nio.charset.StandardCharsets
@@ -26,135 +26,27 @@ task copyXPackPluginProps(type: Copy) {
 }
 project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
 
-// needed to be consistent with ssl host checking
-Object san = new SanEvaluator()
-
 // location of generated keystores and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
-
-// Generate the node's keystore
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
-task createNodeKeyStore(type: LoggedExec) {
-  doFirst {
-    if (nodeKeystore.parentFile.exists() == false) {
-      nodeKeystore.parentFile.mkdirs()
-    }
-    if (nodeKeystore.exists()) {
-      delete nodeKeystore
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-node',
-          '-keystore', nodeKeystore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=smoke-test-plugins-ssl',
-          '-keypass', 'keypass',
-          '-storepass', 'keypass',
-          '-ext', san
-}
-
-// Generate the client's keystore
-File clientKeyStore = new File(keystoreDir, 'test-client.jks')
-task createClientKeyStore(type: LoggedExec) {
-  doFirst {
-    if (clientKeyStore.parentFile.exists() == false) {
-      clientKeyStore.parentFile.mkdirs()
-    }
-    if (clientKeyStore.exists()) {
-      delete clientKeyStore
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-client',
-          '-keystore', clientKeyStore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=smoke-test-plugins-ssl',
-          '-keypass', 'keypass',
-          '-storepass', 'keypass',
-          '-ext', san
-}
-
-// Export the node's certificate
-File nodeCertificate = new File(keystoreDir, 'test-node.cert')
-task exportNodeCertificate(type: LoggedExec) {
-  dependsOn createNodeKeyStore
-  doFirst {
-    if (nodeCertificate.parentFile.exists() == false) {
-      nodeCertificate.parentFile.mkdirs()
-    }
-    if (nodeCertificate.exists()) {
-      delete nodeCertificate
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  args '-export',
-          '-alias', 'test-node',
-          '-keystore', nodeKeystore,
-          '-storepass', 'keypass',
-          '-file', nodeCertificate
-}
-
-// Import the node certificate in the client's keystore
-task importNodeCertificateInClientKeyStore(type: LoggedExec) {
-  dependsOn createClientKeyStore, exportNodeCertificate
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  args '-import',
-          '-alias', 'test-node',
-          '-keystore', clientKeyStore,
-          '-storepass', 'keypass',
-          '-file', nodeCertificate,
-          '-noprompt'
-}
-
-// Export the client's certificate
-File clientCertificate = new File(keystoreDir, 'test-client.cert')
-task exportClientCertificate(type: LoggedExec) {
-  dependsOn createClientKeyStore
-  doFirst {
-    if (clientCertificate.parentFile.exists() == false) {
-      clientCertificate.parentFile.mkdirs()
-    }
-    if (clientCertificate.exists()) {
-      delete clientCertificate
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  args '-export',
-          '-alias', 'test-client',
-          '-keystore', clientKeyStore,
-          '-storepass', 'keypass',
-          '-file', clientCertificate
-}
-
-// Import the client certificate in the node's keystore
-task importClientCertificateInNodeKeyStore(type: LoggedExec) {
-  dependsOn createNodeKeyStore, exportClientCertificate
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  args '-import',
-          '-alias', 'test-client',
-          '-keystore', nodeKeystore,
-          '-storepass', 'keypass',
-          '-file', clientCertificate,
-          '-noprompt'
-}
-
-forbiddenPatterns {
-  exclude '**/*.cert'
-}
+File nodeKeystore = file("$keystoreDir/testnode.jks")
+File nodeKey = file("$keystoreDir/testnode.pem")
+File nodeCert = file("$keystoreDir/testnode.crt")
+File clientKeyStore = file("$keystoreDir/testclient.jks")
+File clientKey = file("$keystoreDir/testclient.pem")
+File clientCert = file("$keystoreDir/testclient.crt")
 
+// Add keystores to test classpath: it expects it there
+task copyKeyCerts(type: Copy) {
+  from('./') {
+      include '*.crt', '*.pem', '*.jks'
+  }
+  into keystoreDir
+}
 // Add keystores to test classpath: it expects it there
 sourceSets.test.resources.srcDir(keystoreDir)
-processTestResources.dependsOn(importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore)
+processTestResources.dependsOn(copyKeyCerts)
 
-integTestCluster.dependsOn(importClientCertificateInNodeKeyStore, importNodeCertificateInClientKeyStore)
+integTestCluster.dependsOn(copyKeyCerts)
 
 ext.pluginsCount = 0
 project(':plugins').getChildProjects().each { pluginName, pluginProject ->
@@ -167,8 +59,7 @@ integTestCluster {
   setting 'xpack.monitoring.collection.interval', '1s'
   setting 'xpack.monitoring.exporters._http.type', 'http'
   setting 'xpack.monitoring.exporters._http.enabled', 'false'
-  setting 'xpack.monitoring.exporters._http.ssl.truststore.path', clientKeyStore.name
-  setting 'xpack.monitoring.exporters._http.ssl.truststore.password', 'keypass'
+  setting 'xpack.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.monitoring.exporters._http.auth.username', 'monitoring_agent'
   setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
   setting 'xpack.monitoring.exporters._http.ssl.verification_mode', 'full'
@@ -176,14 +67,18 @@ integTestCluster {
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.http.ssl.enabled', 'true'
-  setting 'xpack.security.http.ssl.keystore.path', nodeKeystore.name
-  keystoreSetting 'xpack.security.http.ssl.keystore.secure_password', 'keypass'
+  setting 'xpack.security.http.ssl.key', 'testnode.pem'
+  setting 'xpack.security.http.ssl.certificate', 'testnode.crt'
+  keystoreSetting 'xpack.security.http.ssl.secure_key_passphrase', 'testnode'
 
   setting 'xpack.ml.enabled', 'false'
-
-  // copy keystores into config/
+  // copy keystores, keys and certificates into config/
   extraConfigFile nodeKeystore.name, nodeKeystore
+  extraConfigFile nodeKey.name, nodeKey
+  extraConfigFile nodeCert.name, nodeCert
   extraConfigFile clientKeyStore.name, clientKeyStore
+  extraConfigFile clientKey.name, clientKey
+  extraConfigFile clientCert.name, clientCert
 
   setupCommand 'setupTestUser',
                'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
@@ -193,13 +88,12 @@ integTestCluster {
   waitCondition = { NodeInfo node, AntBuilder ant ->
     File tmpFile = new File(node.cwd, 'wait.success')
     KeyStore keyStore = KeyStore.getInstance("JKS");
-    keyStore.load(clientKeyStore.newInputStream(), 'keypass'.toCharArray());
-    KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-    kmf.init(keyStore, 'keypass'.toCharArray());
+    keyStore.load(clientKeyStore.newInputStream(), 'testclient'.toCharArray());
     TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
     tmf.init(keyStore);
+    // We don't need a KeyManager as there won't be client auth required so pass an empty array
     SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
-    sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
+    sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
     for (int i = 0; i < 10; i++) {
       // we use custom wait logic here for HTTPS
       HttpsURLConnection httpURLConnection = null;
@@ -244,160 +138,4 @@ processTestResources {
     inputs.properties(expansions)
     MavenFilteringHack.filter(it, expansions)
   }
-}
-
-/** A lazy evaluator to find the san to use for certificate generation. */
-class SanEvaluator {
-
-  private static String san = null
-
-  String toString() {
-    synchronized (SanEvaluator.class) {
-      if (san == null) {
-        san = getSubjectAlternativeNameString()
-      }
-    }
-    return san
-  }
-
-  // Code stolen from NetworkUtils/InetAddresses/NetworkAddress to support SAN
-  /** Return all interfaces (and subinterfaces) on the system */
-  private static List<NetworkInterface> getInterfaces() throws SocketException {
-    List<NetworkInterface> all = new ArrayList<>();
-    addAllInterfaces(all, Collections.list(NetworkInterface.getNetworkInterfaces()));
-    Collections.sort(all, new Comparator<NetworkInterface>() {
-      @Override
-      public int compare(NetworkInterface left, NetworkInterface right) {
-        return Integer.compare(left.getIndex(), right.getIndex());
-      }
-    });
-    return all;
-  }
-
-  /** Helper for getInterfaces, recursively adds subinterfaces to {@code target} */
-  private static void addAllInterfaces(List<NetworkInterface> target, List<NetworkInterface> level) {
-    if (!level.isEmpty()) {
-      target.addAll(level);
-      for (NetworkInterface intf : level) {
-        addAllInterfaces(target, Collections.list(intf.getSubInterfaces()));
-      }
-    }
-  }
-
-  private static String getSubjectAlternativeNameString() {
-    List<InetAddress> list = new ArrayList<>();
-    for (NetworkInterface intf : getInterfaces()) {
-      if (intf.isUp()) {
-        // NOTE: some operating systems (e.g. BSD stack) assign a link local address to the loopback interface
-        // while technically not a loopback address, some of these treat them as one (e.g. OS X "localhost") so we must too,
-        // otherwise things just won't work out of box. So we include all addresses from loopback interfaces.
-        for (InetAddress address : Collections.list(intf.getInetAddresses())) {
-          if (intf.isLoopback() || address.isLoopbackAddress()) {
-            list.add(address);
-          }
-        }
-      }
-    }
-    if (list.isEmpty()) {
-      throw new IllegalArgumentException("no up-and-running loopback addresses found, got " + getInterfaces());
-    }
-
-    StringBuilder builder = new StringBuilder("san=");
-    for (int i = 0; i < list.size(); i++) {
-      InetAddress address = list.get(i);
-      String hostAddress;
-      if (address instanceof Inet6Address) {
-        hostAddress = compressedIPV6Address((Inet6Address)address);
-      } else {
-        hostAddress = address.getHostAddress();
-      }
-      builder.append("ip:").append(hostAddress);
-      String hostname = address.getHostName();
-      if (hostname.equals(address.getHostAddress()) == false) {
-        builder.append(",dns:").append(hostname);
-      }
-
-      if (i != (list.size() - 1)) {
-        builder.append(",");
-      }
-    }
-
-    return builder.toString();
-  }
-
-  private static String compressedIPV6Address(Inet6Address inet6Address) {
-    byte[] bytes = inet6Address.getAddress();
-    int[] hextets = new int[8];
-    for (int i = 0; i < hextets.length; i++) {
-      hextets[i] =  (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255;
-    }
-    compressLongestRunOfZeroes(hextets);
-    return hextetsToIPv6String(hextets);
-  }
-
-  /**
-   * Identify and mark the longest run of zeroes in an IPv6 address.
-   *
-   * <p>Only runs of two or more hextets are considered.  In case of a tie, the
-   * leftmost run wins.  If a qualifying run is found, its hextets are replaced
-   * by the sentinel value -1.
-   *
-   * @param hextets {@code int[]} mutable array of eight 16-bit hextets
-   */
-  private static void compressLongestRunOfZeroes(int[] hextets) {
-    int bestRunStart = -1;
-    int bestRunLength = -1;
-    int runStart = -1;
-    for (int i = 0; i < hextets.length + 1; i++) {
-      if (i < hextets.length && hextets[i] == 0) {
-        if (runStart < 0) {
-          runStart = i;
-        }
-      } else if (runStart >= 0) {
-        int runLength = i - runStart;
-        if (runLength > bestRunLength) {
-          bestRunStart = runStart;
-          bestRunLength = runLength;
-        }
-        runStart = -1;
-      }
-    }
-    if (bestRunLength >= 2) {
-      Arrays.fill(hextets, bestRunStart, bestRunStart + bestRunLength, -1);
-    }
-  }
-
-  /**
-   * Convert a list of hextets into a human-readable IPv6 address.
-   *
-   * <p>In order for "::" compression to work, the input should contain negative
-   * sentinel values in place of the elided zeroes.
-   *
-   * @param hextets {@code int[]} array of eight 16-bit hextets, or -1s
-   */
-  private static String hextetsToIPv6String(int[] hextets) {
-    /*
-     * While scanning the array, handle these state transitions:
-     *   start->num => "num"     start->gap => "::"
-     *   num->num   => ":num"    num->gap   => "::"
-     *   gap->num   => "num"     gap->gap   => ""
-     */
-    StringBuilder buf = new StringBuilder(39);
-    boolean lastWasNumber = false;
-    for (int i = 0; i < hextets.length; i++) {
-      boolean thisIsNumber = hextets[i] >= 0;
-      if (thisIsNumber) {
-        if (lastWasNumber) {
-          buf.append(':');
-        }
-        buf.append(Integer.toHexString(hextets[i]));
-      } else {
-        if (i == 0 || lastWasNumber) {
-          buf.append("::");
-        }
-      }
-      lastWasNumber = thisIsNumber;
-    }
-    return buf.toString();
-  }
-}
+}
\ No newline at end of file
diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslClientYamlTestSuiteIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslClientYamlTestSuiteIT.java
index 8411a7eb5a4..9cb45472ed3 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslClientYamlTestSuiteIT.java
+++ b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslClientYamlTestSuiteIT.java
@@ -29,7 +29,7 @@ public class SmokeTestPluginsSslClientYamlTestSuiteIT extends ESClientYamlSuiteT
 
     private static final String USER = "test_user";
     private static final String PASS = "x-pack-test-password";
-    private static final String KEYSTORE_PASS = "keypass";
+    private static final String KEYSTORE_PASS = "testnode";
 
     public SmokeTestPluginsSslClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
@@ -45,7 +45,7 @@ public class SmokeTestPluginsSslClientYamlTestSuiteIT extends ESClientYamlSuiteT
     @BeforeClass
     public static void getKeyStore() {
       try {
-          keyStore = PathUtils.get(SmokeTestPluginsSslClientYamlTestSuiteIT.class.getResource("/test-node.jks").toURI());
+          keyStore = PathUtils.get(SmokeTestPluginsSslClientYamlTestSuiteIT.class.getResource("/testnode.jks").toURI());
       } catch (URISyntaxException e) {
           throw new ElasticsearchException("exception while reading the store", e);
       }
diff --git a/x-pack/qa/smoke-test-plugins-ssl/testclient.crt b/x-pack/qa/smoke-test-plugins-ssl/testclient.crt
new file mode 100644
index 00000000000..18221208c16
--- /dev/null
+++ b/x-pack/qa/smoke-test-plugins-ssl/testclient.crt
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID1zCCAr+gAwIBAgIJALnUl/KSS74pMA0GCSqGSIb3DQEBCwUAMEoxDDAKBgNV
+BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEiMCAGA1UEAxMZRWxhc3Rp
+Y3NlYXJjaCBUZXN0IENsaWVudDAeFw0xNTA5MjMxODUyNTVaFw0xOTA5MjIxODUy
+NTVaMEoxDDAKBgNVBAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEiMCAG
+A1UEAxMZRWxhc3RpY3NlYXJjaCBUZXN0IENsaWVudDCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAMKm+P6vDAff0c6BWKGdhnYoNl9HijLIgfU3d9CQcqKt
+wT+yUW3DPSVjIfaLmDIGj6Hl8jTHWPB7ZP4fzhrPi6m4qlRGclJMECBuNASZFiPD
+tEDv3msoeqOKQet6n7PZvgpWM7hxYZO4P1aMKJtRsFAdvBAdZUnv0spR5G4UZTHz
+SKmMeanIKFkLaD0XVKiLQu9/z9M6roDQeAEoCJ/8JsanG8ih2ymfPHIZuNyYIOrV
+ekHN2zU6bnVn8/PCeZSjS6h5xYw+Jl5gzGI/n+F5CZ+THoH8pM4pGp6xRVzpiH12
+gvERGwgSIDXdn/+uZZj+4lE7n2ENRSOt5KcOGG99r60CAwEAAaOBvzCBvDAJBgNV
+HRMEAjAAMB0GA1UdDgQWBBSSFhBXNp7AaNrHdlgCV0mCEzt7ajCBjwYDVR0RBIGH
+MIGEgglsb2NhbGhvc3SCFWxvY2FsaG9zdC5sb2NhbGRvbWFpboIKbG9jYWxob3N0
+NIIXbG9jYWxob3N0NC5sb2NhbGRvbWFpbjSCCmxvY2FsaG9zdDaCF2xvY2FsaG9z
+dDYubG9jYWxkb21haW42hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3
+DQEBCwUAA4IBAQANvAkddfLxn4/BCY4LY/1ET3d7ZRldjFTyjjHRYJ3CYBXWVahM
+skLxIcFNca8YjKfXoX8mcK+NQK/dAbGHXqk76yMlkrKjh1OQiZ1YAX5ryYerGrZ9
+9N3E9wnbn72bW3iumoLlqmTWlHEpMI0Ql6J75BQLTgKHxCPupVA5sTbWkKwGjXXA
+i84rUlzhDJOR8jk3/7ct0iZO8Hk6AWMcNix5Wka3IDGUXuEVevYRlxgVyCxcnZWC
+7JWREpar5aIPQFkY6VCEglxwUyXbHZw5T/u6XaKKnS7gz8RiwRh68ddSQJeEHi5e
+4onUD7bOCJgfsiUwdiCkDbfN9Yum8OIpmBRs
+-----END CERTIFICATE-----
diff --git a/x-pack/qa/smoke-test-plugins-ssl/testclient.jks b/x-pack/qa/smoke-test-plugins-ssl/testclient.jks
new file mode 100644
index 0000000000000000000000000000000000000000..d6dc21c1bd5ff4a248bfba5a171518cd649e278d
GIT binary patch
literal 3358
zcmdUxcTiLL7RPgw&_WA@Dgpr%0Ra<wK&41qK_ZAqQ%VvMjYO(+NI(z-q^NX5FD?*@
z3JTKOiVD(Hnkb-16+{pr;l<s3yS$w@v+s}n=iQm#%sJ<tnR9=?b3WhuV=OWjK_C$L
z_XU7pJ_K(c9MPTN;R6Bz*sb^jWgrj>2t|Qzqj*?h2U!6qpaA9r02UCG0-ZeoW^;$=
z+VdV+;l3t{Je2F^&mzpV-Ik55-yc4+k?XPK74>5FEv@hI#YMj$eD{OMwLP6Xrpd6|
z{K$HxQI)Ou(iWbPC+jt6?CQRb5*)aEfGniRI5FqcLHiQ;p}o(jZ|7`6aNfJpe13!k
zCh7E`_Y&L)^3WAuS~w<KghR(z=XQx7oRU;w4X(^ZRw-IIXhw_mN*IYnavGQR%G7L|
z)EG-*KeW<(yW88$!qdmWpt>b)`b`lJNt|enT>{qi$;!lIjE|g3$7jVDsW_IQptJ4P
zL_KSa{2N0Jo2a72u<i<{`j?f3X<trP^y{%#D+(E;Vp5&`mwft6rlQapPVAl}w%6{+
zQrr~-@}-&Um<2#2M6yZ|rQc07)wzQnxoY-Ni39{03<Y=4)n<(rCy68QSA3;j&vP)D
zIQ5~Oo<|e63YSgpMJW3Q(0l5N3KR>pU2=xfl{80=FZCD7r1Ku3?_azxk;M5RFzjs_
zKf_4kUTx2YVRdXErDb5s1{7=fY2m5W<U;HlzJSXR!)6KC5yoDr8Eb{swbwwFl<!P)
z8OP$jV=(MB49%Ft+InsutK-40HGBW$12w9h+H*b<xWfrvVWHpdp~9m_BS0tJzz(a9
zM3!5b8BdRrHRxLYo2CoLLq~9BDKX3Cjz1;H82h<c3lZFHPPDFNrj=Klq_Kr8jHJU`
z)99f--}Ev?W|XX6j_4<1T?gX^Y=%8-9GnIC5;u60WW&u4!&=M{sS%&@3t-8XZ%040
zq*m^X<y&@0#nT6-Q58cgp?f#fI=`i?k5MljYC`Lkx7v-`%{IxetrHnT^IXe@&|U;I
z;sO8h&U4>~TP+I*SI_eDqkVFgr^#K@jnGp8M7_5B%id!j{Rv&0zj!Yy;vdaaEXaRK
z3Yps5erJTjW`WDhK5+_Bg^lZ*JhHFtL&Ev+_n=EvycTOiJoKoHrJEbqjT3Y~`9*!N
zAXfTHDe>tzYbk1m>yKNp%I5EoKuh3LB4BuxdYz1-ssS1NdTU_(g8ciq+SD-2ZQ0pz
z(w#ktO)m4g<{~|xKN7B%NVRFXt=($KqN6oS2-CzB{pmOM{<v@|Xmw7@TeQscp5CjZ
z^Och1Fe$Bdu4d%Xld79oC%@tJ6kAWRJ)5+5y-wBV>jS05<eW$8<}h|oNRRIu8Tg)H
zakV~nDn!FA<8m<`LzoFh#XbROO9n!ZxE3%a9}l=i9oF7#^5Lm;><w_(=4iBuoL%g!
zv#zkq)i$X2W+VAlH7oOh-UAccex?QbmRfEWY<Ty=_64Phb~d@68#i;hPxpcoEFGr?
za@zYxhO#N1R?6lV^;<`i5$JRei3L%y3R~^Ouf)46OEF(J)Nt7av+?Gr;$r-mti9Lw
z`1GQ_g7xir?(=0-;iOdwE>_5y;w$U#8aSND=qq=4Q)zH9-M)(krDn_0B6!A2ffDf6
zs>-_VHI(yI-Q=9Y>n%IESBnAh?e|v&FmpY#K>oA4jKNz^W>yV&92Jvgx2o1@jS5z2
zHpH;V*6Qv_xdL;$(rmY5$EH>{mBfk#%GeK$Q_oH<b$BTbdR8o-*>brQU}P@M#zQ=j
zQH$*F>*;VF(7G3_WU|N;`f&97%LF)%h0ze;TWAzar`HBTqn$Y(pQx}HeoB$H)tb5c
z@fDh>j{u0pR#_c|0f8W+C<^2iiUMxRfq(%pm;>}^Fm*NAtWh4t&B~z=Wg4}?4FH@>
z)jW>oLcv%er=c(egyiLh7D5U9@Hi3N1fq+#k2}tr;Npe5fR;uff5agOQKQ{)q%~8F
zk;i^jW0Zt2H=0>qMHQ{3uA;7P&vY?NSLJv2AFV|Jq<3$HDfSQwz{~s#05C8G06?up
z-xxG5wzn@|kZp6bqx|GFoJ?X>y2)QPul6N+<(0SSR$E?fJF*`qwSFT_h4p6c%&J<K
z?dKpoL-NI*-W#QL_pFV*tjys^4>f4IkWAZsz2%8Za)J4=`tyO*nm3Iw8`U~bmy|kP
zn>e{l%gU4D4RCS7Ps{yJEvG#M2<R`SrEvkJ-EwxE7mf&9m)tN|4(lDzt_bh*2jtkP
z+p?X-V%@o;^3=m#qIF|w$ho0F{pX|V+8(}cU%s>k+{!mA3FwGBDtpfPnX4}KT>u9)
zMS{Ft@Irpi-6|u;PuD~IC@Xwo?EFaeaq3P5A&v3gQitloZ6s4ZUCbjw3Za#QApi)-
zCpV$U4JZy~=fn|EFbaec|6%e#g`fh-Lhv&hcbhMabotqW&-_F|=mcFvk#GKZ&j%$(
zqsWmI4k8KXLcBoo_Mr&=VJiP>;YpWW+&w6;Kf-Dh;Xm!&IcmF-G<GFv>`u~%hK4cc
zHyX~I;@uChyT{CNW%d{VaW`;?`>w7~Z?<qGaN_8O*sDPV(X(;Zs|o0r&e^Tbg2Sgv
z%&QGnq*_jR(nR8lU*(3$UZKV7(Z+$Q=yRny^D_IBtMj8zCB|gi0-=|FiT-)d!;rP{
z_6?3v>XS?d|B4LC%su?@El+t=JUlfoXj*{N0vz2gvs8EzQ>8JSSjHOf+kE52K`Y01
zTq#Mb7|oqprG8n9&jH#1Zm))7fW2`o5`F93JHfzpzElyxZbirJ+mwadN&I(yp2_3Y
zvlIDrGLqtW<<$OB@hl8>>+#vV*lgv=-geg(k-(K#R(h$C63XY^#|-j5e8HY3S-l_S
zhb-W(eg5@E(dYN_X#zy%0f9m6KW;%DBs}4_JCOR_KiOw;4;f&xU;l6R>mCs`r&ONx
z{N*rFrvD@RNR-5H><j-F_AQv-@E7tm|C4-8`+u;~-*G=tl|IC};DC**SLj#9Npe}0
zD~hrUk_7vq9*?V>2fxDP27Oh^%h8T9@ni&m9mJW&;uoeU>%0o^0cU~5ps&#;HpeGr
z1)J{oIek+S1e(apV@r#QO5GvFwWC*;yUt7ZGb#=7DsoZZ1e$a{LV6RVVm`k&t5UeR
z8S@w<rL3NO3duN$$m|FjtPz>-E{>Y`N>7L908h#7A-!?&u50S{c~PIy4f2T8J8uXR
z?ep@F0-mM0&6i8m(CxA%ikmnCJAG18H{<tqPlwVSYGQw3WYv~zyLyiBONTvudVSCf
z7!nSXtmAsf0UeLGd=zqlenYp&VD-b>xiYCpapAw^{ur0-gTx!(4v%fmh{uN81bIz2
zcKv^F-`y$dv#l|H+2d?fvUGXI!6<}_%zTluTj;RqZC7{1F>9}xh!+DKEfXmh$90y<
zgbKt+Xvr3C47R~I-J$-Whrc8^&UjYtSn%Fj<37$Ts9i*8PV=&v%wtuOtQ$+_I|s|x
zkzTetw4lVU7S$t}f*Es_1Th_WNoZPPhF#v_xs=UBi`k>N&d79o1;dDE60{4X=_jQV
zgko%{%CLO%#(=KmbIk>F{#ao<QnS%_<7>F-h!qdmlIrlw^RV1W^^QE6s_ginyl|6j
z9o8)(pZvO`ueni>;*0New^Q?*Ho3G@+opogGU&Ok?WxK&erQl-B5X4cc%7(>Dy!g~
ZL>oJOr20!OUT)aV(#sM_Gi@+F_b&|~n(_bu

literal 0
HcmV?d00001

diff --git a/x-pack/qa/smoke-test-plugins-ssl/testclient.pem b/x-pack/qa/smoke-test-plugins-ssl/testclient.pem
new file mode 100644
index 00000000000..7268c55dba9
--- /dev/null
+++ b/x-pack/qa/smoke-test-plugins-ssl/testclient.pem
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,C98A45E4AFC263C2
+
+wLuUEXldYc54r4ryWd6jw6UMGYwn6+ibGKHp4sD92l42lmI2UrCT/Mb/E0O+KMMy
+pHgc5/dBWkXgMiqDyLIhHk4kgT40rdw5W5lZkAA4Qt/Yzd+rbscTvzp09zrF6Fll
+czgoE7FrvhOKiEOakerTit4pIPYosdX606cpVQE2lq9oZs9HVMcLzdAZj8A/P/4g
+fo4X3+zqVYC/LH4n00bhNoeeej2o1lEJ+l9u9hptT2ATXle6pANa83Ldg4OxJyj8
+dkR9ahnAMCvYTSjEU7nwmGNPeFX0PIUjJKQivr410cYG104DC30Yy+XrIUfjTVUi
+agwlMpHoBq79/ZRUJR3xPLkIGgw4g+RPt45D9eKsEsV4vqy8SFlgaoJ2mKUKleZy
+i7D9ouzMKQ3sYE4eQVQ5o3K8ZPn5eozCwCVIp7jGSsuvDpLA9peZSwWPfc5y8JFD
+/64usCt1J8Mv/e9NVllC8ZA+ZmDitTiwLZysczpMOaFqqeUbk9EJst38n4nBzRV2
+quxvg9W/iveQIydFyftCtNfRkpbp0NCsLz293dBYwZacHsPcY27IBCwXHiICjiAW
+q7bnisXsgSaQMhMNRGW9YElZGb7ZWxoIzcyNBisGI8zxn48ObERVOmkOFxY/gs9T
+YmpVMliWtmRG6hb6iCh9b7z8THRquxgTGE9ZFBwtLUKg33aubtgAfnUh/Xq2Ue5K
+l+ZCqDGEi/FSIjVENUNNntAx/vXeNPbkoGLb/HSJwAh+sjpaLGQ54xixCtE9l3NY
+o2QAiZ804KLPaGtbbOv7wPumxQ+8mxG5FN0hTRrsMW9t8pBXw47iMy/T2H21TD5D
+E5XbM6kFeBrnsWnZJ2/ieXqDE4SX0tm3WEvZlDg7N7jV8QDM/D3Xdkb/sqJRabMG
+tQRgwkLiB+mZ5MAfGLogI2/lOEayrBVz4qYdXojewxY4LtaZ5HiUIlyA9CJelMvD
+nS52I6+FpaFhvuZC10qaM9Ph9TNyx+XKRUsPILuDiBRnYiHUKs1qASl5tjn2yyjM
+71WSo7A7btOckzhDZdMVf1T472f0LGsRYoQebMhotqCuR7yArZHzTeWB0CjL3tOz
+j3QlhKt2E1jx43bSK5tBasd9Bpmn2onvdwu1RRP8cyQBsXJSDy4/8t/g63+C3wod
+8VPrlKhK+TenK9EoEqJ2mNuNq+duOjTXfK/7GM5s0BFKv+i2ckpDi1NPckd2gXjF
+yUFZhmK6k0WC4jjWloMt+WQpi1rXMEXwCypgTrqWbvD0p6+X3uQmP57L4yHQcZoW
+Qcs5GnihJ0DIhw9vYDhBhNo0WY1oBO20nVCN3R/JIpp3uDtg64WvfvMSXzJIPBCY
+s+/GM5TtuD6mERDu3+qXxWwiy4PMQRcgjRTMEZ3A4Iv77YfQRkcd6S9qjUUuR/5D
+xs+J4ryb1biz9ofW7I+Dbz4SArWSgwcuh14AV9RBv6Rh9m83rjT2K0yvbe/+7hHW
+R8nzRMqJcGNGCHmRjA/cwoiv6+k2J/RbCJqnR3RmNex/85XaXBfZwRfHXVbzZQfa
+SrFaaNLf1hMwGLAJjIcQRxa3yZbjFXVx1Bp4hh8rKNWaOItjavNtNg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/qa/smoke-test-plugins-ssl/testnode.crt b/x-pack/qa/smoke-test-plugins-ssl/testnode.crt
new file mode 100644
index 00000000000..08c160bcea5
--- /dev/null
+++ b/x-pack/qa/smoke-test-plugins-ssl/testnode.crt
@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID0zCCArugAwIBAgIJALi5bDfjMszLMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV
+BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEgMB4GA1UEAxMXRWxhc3Rp
+Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTUwOTIzMTg1MjU3WhcNMTkwOTIyMTg1MjU3
+WjBIMQwwCgYDVQQKEwNvcmcxFjAUBgNVBAsTDWVsYXN0aWNzZWFyY2gxIDAeBgNV
+BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEA3rGZ1QbsW0+MuyrSLmMfDFKtLBkIFW8V0gRuurFg1PUKKNR1
+Mq2tMVwjjYETAU/UY0iKZOzjgvYPKhDTYBTte/WHR1ZK4CYVv7TQX/gtFQG/ge/c
+7u0sLch9p7fbd+/HZiLS/rBEZDIohvgUvzvnA8+OIYnw4kuxKo/5iboAIS41klMg
+/lATm8V71LMY68inht71/ZkQoAHKgcR9z4yNYvQ1WqKG8DG8KROXltll3sTrKbl5
+zJhn660es/1ZnR6nvwt6xnSTl/mNHMjkfv1bs4rJ/py3qPxicdoSIn/KyojUcgHV
+F38fuAy2CQTdjVG5fWj9iz+mQvLm3+qsIYQdFwIDAQABo4G/MIG8MAkGA1UdEwQC
+MAAwHQYDVR0OBBYEFEMMWLWQi/g83PzlHYqAVnty5L7HMIGPBgNVHREEgYcwgYSC
+CWxvY2FsaG9zdIIVbG9jYWxob3N0LmxvY2FsZG9tYWluggpsb2NhbGhvc3Q0ghds
+b2NhbGhvc3Q0LmxvY2FsZG9tYWluNIIKbG9jYWxob3N0NoIXbG9jYWxob3N0Ni5s
+b2NhbGRvbWFpbjaHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQEL
+BQADggEBAMjGGXT8Nt1tbl2GkiKtmiuGE2Ej66YuZ37WSJViaRNDVHLlg87TCcHe
+k2rdO+6sFqQbbzEfwQ05T7xGmVu7tm54HwKMRugoQ3wct0bQC5wEWYN+oMDvSyO6
+M28mZwWb4VtR2IRyWP+ve5DHwTM9mxWa6rBlGzsQqH6YkJpZojzqk/mQTug+Y8aE
+mVoqRIPMHq9ob+S9qd5lp09+MtYpwPfTPx/NN+xMEooXWW/ARfpGhWPkg/FuCu4z
+1tFmCqHgNcWirzMm3dQpF78muE9ng6OB2MXQwL4VgnVkxmlZNHbkR2v/t8MyZJxC
+y4g6cTMM3S/UMt5/+aIB2JAuMKyuD+A=
+-----END CERTIFICATE-----
diff --git a/x-pack/qa/smoke-test-plugins-ssl/testnode.jks b/x-pack/qa/smoke-test-plugins-ssl/testnode.jks
new file mode 100644
index 0000000000000000000000000000000000000000..ebe6146124e8fd607e46a1a3129bdf9b4de0370d
GIT binary patch
literal 9360
zcmdUV2T)YowrzLQl93!F2SGC3WF$w)IfLYGK$2vT*iDigl`N7oh@eEt83BokWI;th
zKok%V_zil_#dF{N=hmxN@83^VSMRxd)mp3enq$l{$L93&>1O}{0788*FTTk=VD28y
zE;cYOYe##Svj^8*Hy60QBMbllY8=Nx4**vTxdk91003Gv1c?>_L4sT!qk(`R5Eh_)
zU50G!4F3%T4;_m=TrvCr4+zA$0)Q|==^?b}XnJ5Y;(s+egq9o+ikg;}4=OCcD<Ei$
z3Pn&M@2~LRnga=Byu38<3IGiW#0Nl-Kx_~a2m~ze=_P%T5mh>IVy2IuGcP39Gn>U%
zK!zH|Y#zr76z$THNrVtl2Hz*duE|_x{SHKW=Wy9~#x0?_!WxzreKbBd_{B!%FFc+}
z#hS1iA!Ph9s*KL}5ZCWMX)1FD)R@OQManE3e-@+qd0nOC5~I@Wvv)K719(Ez^^D!&
zt6;h^oFHXs|H>4zGnJ%}f+`Pp>rEdE;=+o<-25$T9lIy1B0c;S!Z_r&_k6nwupM+P
z`QIj_3i+10?X14(vmJKmiZElETC3=_VBSWbe19M>na%UTg<AM%!AMDa299~-Qy;h2
z$8;?*a!Y$t(NgjUQLChqQl7TNC5_2AFBwF!KxjY!FdxwdL9{@yQ0GBI3<f~}5SsJL
zt6(xP$wWtE$|wWR8Qq3SaMJCMEm^xnrw~N^`R{~aL<9s8io|ksv9@%yb8+`TlKy$g
zeQ{;u;$&&>jKu!)Hb0X5AJ>=1@Lw7vcxjN}<w1fG;2>1eMG&CU^70?>@)@Jj3w6do
zK=}wbX`7exBW2Hy$yVaDo$ICp=}fYb@;nQCWlE<RN43Ht&AP6m9$7nW|4BMEfXy^?
ziF&zcj|JZ`Q~71o8J?{L?5+WW+@lQ`V=q~4)O2U_o7{jPjFE>e6F`UeJ?OsbuRMGQ
zgd{VQ1`}qlEo37p4u-{8N|I&os?=jM_S_F93rh*dwGvc4>^v;F@<@1%AfI?JI<wYx
zpPl(x&bSyG+8HO`BzH;H0F<G+Gk|eBy3qN<$|DXhkKPR}r6!Xq$DFHoIQh$LA$=2;
z6c*0Byh`yWJmWSdJD-o^s0|{~L+W$aPNTHAEe@HIIXLE{Z}#4CC{ewQ>xoOG#d{a^
zfIt8&OKT5%FPM$J8|t~~u}>Z;{sXMDD6qapfwk=?SP2aA8Y;D$$)co!mluiHCH4RE
zP$7X(VhABRHdYc#Wb!Vj4e%Nnnzf4)lm<fe+f6)}qouoty|p{c(#_fqN&+GJ?Pu^g
znLvpk1i#${S=vBpA=Ez^fz;B5OI}r4M^{N!M_yW6R#94xOIG@V5BMR%yh2bxUS57)
z6d&-4{0#Zd!~f^|`#Y|wRfD$=92q(iAIUJv!pf_A`fv`!)ndM@1t~o%=TtFwdLgCm
zd$J=2jZ4U%P$e2`_S!i8T17ZG5UAonoitlR_~mIS7gIxNkN!(uf@~d|8SjqD(6*cS
zO%gmqT|WxzkGqdQVHhH%-9wt9*EKMhV)EHlKQ;JG%dHkvUsXNI`R)^6tQ-|amix)m
zsZ>ZJVJR-@b8DwGjw9y}xw)Rb-X`tOZ@L?r7m{x$tD>pf-DK8J8PK&?Rve%UmMWfn
zkY1J8$6e`6OS^j3!ZBdDpQ)-!J@JJ984<Da*?f&={VPYCAFso`4x;4ez^Vi};zA!D
zA$VGoB|6K57;Vin(Q<!sjTS@$f^$wE_+T)qNQ6W1&jVs0@C<wff`<CKL_P>?Dw%90
z9h<CTT)4tSN|~ro7ZWORusG0xU?5V66nH)Z;4gW40e%Z73j^^QTvZiDJ;$|Po1NDn
zDMfOPJJk2EF}a^Z%jUrOF5GO{n#t|zP!A|+zgE$`0qx?&=kq=tjU{Ee$A(>9Kf+y~
zZ;X4CM_eV}GZ5(`<XU+`quJ30Z|)NYs$dAP(t4;if7==|MYk{%+@{Q02yM6L^unCo
zxj$x5D5Ud_hvaR4g<VcVgk&U{>=K*hqpZ8TuPP9ObD5GdI!|KXQ!DE#8?8q-!J#9&
z<{0CP?#6ZZ#oFo@=oP4^3J28dh2~!eXSx~~6ipLu^f&9MPd~6p^1dZo&i#yysyksD
zk}_18V>Qk~vQwzp^Jc<7;yAcH?9kbl>dg;vDb^QW7vRS}huY7Q{j*f_g#JtPV*wf`
zQ+Jb8TG{^cI3dbVTnP3#FtLfzT-<D-WDt_`2#5GDKxc%|pWjC#z9xT}0O)m5o1mAy
z5DtGS=>or(bOGc4(Y5{#j-5r{Ppfe;7RN>qhPm0{UaW%Vx1xCa5qm=3uM*tyDmx^f
zYB_a@Gg~w4N2l?k$LGG^<?l1x@v}Ll8=)MHE^R2&RdCZ*C7^fa2d9%Ub=662FF3H?
z%a4-YxR+Bq(~7On*WhZI+#snR!<wn}M3bh4fCi?tJwB+l>P!NIepM`u@h$CVHNvqI
zC)X{Bme~#(ow!*Q{K^N&ikb6+rMH->KX;ZbN6hV7!(1Bu`qVw?;aP#3&Yrekzjpej
z<g1kU_QXiAm|48ElFV85#mY&hMf@ljVWWIhBX9cYet;M93n3LI5xu~C&RG>K?ev|N
zXpSYGJX7Ur@l^_!fa=P>E&WMk1O|eS+U?%<c^QHXl#s-tes})=DE)XXSTvryUvlC*
zu;Os6PvkVb{b1K_$LQ|HLB}kzJ1t13^-5Hq%6wtyP;;%Oh$()Z8^m(AI##NB9#|7$
zRw}x|#GLdrKSC!VGTRUs=r9mbPWe3G+kE!{)@;s;OcS4~4CMPVo9Pr+Hb^W%YM$RJ
z3644_qMK=}P*bEva5|v^J=U{5dW1vUbQw1}aaTm>te$I}MSaKjCeWH%kki*#p`IR^
zVzx|rZ=W!gg0!F0H2VQ^^Fbof!}9le_)<m`pEN^}rgwFiXKAuTG>)5Y=S5|6FO7Cv
zbx_><@>*LeHI$ay>|Nv}{__#cG`gqE5HI=yy!zq2=%Sr>Y-uEp7wE@yxeIf4w6R19
z0pRVhFITw##6I{Y1PK;IRe#z)WcoxH4#%|zmFAbRe+4DeU7$D@vJi|;4ARs2BfumE
z$x8ph0~o}3YA&A69w?nhuWt|Yh7v%o{$vY`Up(-~NEBoI#tcxx3+_iT2BrpTtEzI+
znim8i1QFsDfC}<Mq5S7H{ygOS75?vU=<ir$(A`@zo%ujE`O2$Ylf_Rh-@j_K3}@HL
zkq%468BjG`VX*gsktmjN8@;>TOWK!IHlvq*$BEG}@fH_Cm@4K8Uq-IvD*m|7@F6Jq
zuw<y(Y0M0MLTEE#GJlmj%d_(v(2r*#A>GO-ZesCM!egNa*#)E@4QIrW8I23(sbgFL
z-JqdF#j`8Vy<a5VAV}W6@2Ax;#;ysN0@cDSOr@d{Il<(6A8#uBC@RuUTuM;X$~3-?
z<j)f0%TxgqICe3^2*QoOafB!w9`U5uDBOsQiK>}VmdE^%g2k0bzk-aO3BahcB5rNS
zxl7R4(8SoexacwQK1c1P8)rsVPYAtP{l+gAnS>z5E){N%TZk`R&*4@b<AqKOnACfg
z@YZYyA{UhxG)!PbIt1}BnB>12lj~mK`oDh0NG!mg`Hy?PMdCk{$fc$gg@y}Zfu}F?
zSu0-A5J<7i9~u3DF0|o^$U)%UsmE_ku>)^Gq%Eedjl|h!miFi4Sf0Lza4Yh$3`(a-
zlSoW|GYZ|6+ksyVU2ZqAPbBTjepxQ8f5SX?^^wnbkC^~H#@eED&|5^5(CZW?&81lp
zuK<Bio2m%4`W%&oQPP@V;BF<0InnxZ#k(_fM<UC3(7qt0Oy^9?&}xCSmxTiV$nvOg
z0D!2M-OV?wXbK<G8cCi;-8ka$TqNxH5PK`;kw3-JiyfZ%*rg3(edmpl^)I@rEhVSL
zK1U}I<;f78fSFsh%%5qR23ooN)Zvl$h%K0Lu!uya#1)U*1>(Kl>+YUJ=bsA;AOPnN
zL%<wnjf$<EJhtzleh7jDps<g9q0ZPrgg{ic4R$EMdh$%KJD#W>!JXclJ}L<W1goje
z&5)Y`nq3nE4Gx&D`n>X4jeGrIbUTc!v*~SOh1OiRa(L7uS+&TRqzL$=2m37@-?ugp
z;<967zk5`#GF$mwu!!%bj<Pfl5^^30J&L;?QlQ^SW>bE#w?DiH;8#?Wzm57N|0FrE
zn;pF<1Q#6>WkZaJqlDvMT!`PO>8EH>M_C>hj3f*ZfC@oG_ynO46eIBqKp_y`U*T^q
z#J`%u2ttgGaj`B4_-g}bKnye)0C)>_&44m!Or)#1Ut8&;!v0wFl7XiLS&Hkpf6yQ+
zS$u^RL1(ei^U17X7FKiWIm_PUG%*qPkB-&%nj|U<+I{m8br3|&CFH3AtedxB3{(XP
zEn%<JnoP;D%xC@si}r8lfTjq5$b;x@-FUp<BAf;$y<yt5WqNPNeGfH+=2)F4KDbN7
z9W&^G7yx5^T6$4a&MztNbdo$g%y6i;QIRl`c{c*Y#8$XU$2fhF0l0r=r<=PaDm|?x
zO#^iSfGcRIs(1EJ35o=7Je;#K-hE@Ow9yZsE4!}{__@p10o|GMLm#^J3E);u%{_nZ
zar!v8*9s{k=OQd}Syr^I^~V_B+LAjwUy#As<@=`B^{t)uH*>Zcf>g8WD}%`mB@~WM
zp9uG(2Oy&6>W}bkjBJFW3kT}F6e9aa)BSXsD4R<r#yoYWDvW}|`_!7eing(FF^Dw<
z0q%WmKkA)nU@i{1s|Lsxx&yuG=2p@J7DXGS=I>Z=aNVx{s&pychT8+Y$$l_!H^FaL
z3LLd5q?EMk(g;f9FP8|X)yaCx6I!~n)nmu%<bOsF4HOg6G&Hv?3aa@r2*jrXUxU@m
zr<RdRYf?C0ovYy%&lp*-!<3k^C`1GUuBA6~WcHfvz<CI>`$=|;KQj#O^GGWI)jJJs
z7>Ac9vNah6&K&AwA1hzS{iHejdVYVmZF*^BM|8K!2)nP`sY>CxeFi~sC&l{Y_0{O-
zR3w-dQMDW`5;v=7NA6>dI8VL&`0&n)2=-n!aIaAxvy^GPc>jC8J0k!|&vN;X?e2o?
zwky$-nw!ods}&g=K8Qr_j^(ku%x(XbPEm^7#X#04E>m|5a#SCvn|mjcA-(GfSo!6&
zwo4GA-&byR`#aIaH8CYDzQA5wsTt$tD+m|9?pO*JTVtH)G_p-^rk-c&&hGLBJ0dr&
z2HYnan<c3~4w^U1IdC3w_2M$iTW0D@f;76wP8_hpV5P|i%GfcKTHUqn>dk_~8S)LW
z{)anZ&dF##-IbO!HlOrrA1oi$i|ZKRW2P#NW12=pa%4>IZtA*obM<WukLhs2<@?>+
zUr`Tno4i}q5^Kd+o7JSL=5rZPlc46PloNW-aFxv_(_h*;*fV`daXfEiSnv3Vr)ub+
zGvw0>%{HX26_R#m)1Q}ov&mmtWdu>LrUjHJ2jK4u9jC@di^dq<jC`l`Bv9;(itmA0
z9r(z_IlaKWG2w26ev@YLOuhI>=M4hG)N(@7*Ibn94fDkgeZ1{bnK4+j=-Bc-b2kbl
zB2~`9MlJg3nM9CMwMj#DMNZkY^(!}d=9l~B)2~-t=jY-H@fnybqV&{T6)llgafjWi
zre8Oej@qni0pGukP;yElQPyUbR0vj64{6xhQ&{gaRu9*}FbDPDrKEY?oF2X`^?(_4
zmb)GR`9|N;k_xAaOKOcB-X5VI4lS)VpXq&hy5>;c=;P=Toe{dSmZd>q!5ROshhh@8
zezz8bnn9YgLPmvb&zHYGpx{ZvsYY(j<U>gBa_`6ML+^wSF{M8|!4r?N-PPovp{FEv
z=Pcib1<|(w1v)7kIBo9?+~*w?elzPllgNPEn;f9$IlZoM!$#4pBvCIflR?r-u)c6+
z(<;2~Ya?~aow*x1$ftJceu3cv-rdSCcqt5`!-iUS;QcI~^>3#oN1~~|xtRM0-|ar2
z-}HJmdu!h-8}h(C+}L*|A=$sKNQ*=nCXxLHm$0Of+U)vQ=e8;`-id27tf^>6nRV(o
zD_35LMQCiXvc;2P5C$3ZExN#sI331)j7wM4?A}WR5y0vQHB4^!9Cb#LW2-5h8Wg_W
zE?(7>&&`t(tsSfKU2gO8^6pwlf#)k(@H#E`*xxz}(ZGA`$b|V!H-7Eo_a;mxUQoaG
z@iUWo&dy;${A)aMi66W&;E|)TL`@ITKe|!zaenPBdhYvMi9C1z&?Zn0VDle+KtrRW
z&<gL%q04>1xm~aLZ+Za#F!ldcDg52k7y8`+5HkKBT<PzW`$A3n6#Ax#MocsNIJY$&
zu68BoHB3?$(s8hJQ;o&s9yaTwCvRmX)RZX}K@8NGv{sC=*<3;H<FgY?Sde`7+l#oR
zue`)U+E(|<{5uyZunn<2xAmEmt3RN)e%5=t56(D#`b5r#mo@x5Nt@_Lw9z<*$enj8
zHSF<+kxc*wZh<5n`cqBf%pSkVT8fSS;_!vNlXQZ|z(GWJz-UaY)i(j-yzm`p3mb9j
z!x`8@_Xb;|@5?mXjY`_u6QgX};x?Rny&lP_hq2WCtAQsbwNV47S@k79tX$s`F$N6|
z-k)>>PLT)EHQ+wS0?)^4H3rz7L`xRQ?0#5WuV4tJA^+R{Ul!N!Swi%8vAG}bX`+Jl
z{oGbt`~Cx^#3hmbhxOmzd(GpA;JlOb?eHYV$_$QhVoRotB5vEjX~hRt_QbNfZtp`z
zCa^jdlJCrmZdH&KP`N<qI`Bj^S`^Yvnx8xS(1BtU)>vizsp}P9;bei0LINMRZ>umh
z@wu?rUddcG(Rvf=W_VWZm(bV2C!R@~vHk=`B}z~dn3j-Xlqa^He3+oVCSlzhnr_T4
z7xI#}+RkP5Md<>pSR;^knyvj{LXvJ+Xj7FairmPhUH(WR%z8ECi!=5X-}D$9J9kN-
zC$E~1Wqy*4yp5$n!!{%z@uufhdn+l@)27$nh~I1Vmcv<n7q3m0%+URtu6(%jJd?Z&
zL5F$3HwoO3iYoji6v{z>D^9)+Fcjc{mRqB-6n{4VXay+HKSqK6{!h@8K($I}H)GZL
zd+vn7OSEuCTdiM0|B8QpC@%zZ-qoWKv;9Igy)4Yl!yayLZRr7{*L87mq?h*eaIu9s
z!`x7JZ0LU(_QDWeC`vr@i$J00%{&wp^8X4Ugiw6cLKm~5yIVS0S@NUg`v3iXT_puI
zWzN=ski(8IGp7|TpM;&i+%EXYl{BUj)_2s7CuR6q-a0@K^u0%!{YEWpYsrF1XERyZ
zf$tMSTink}o&$ffH!`}Es;V{6M4j$g-H?#|zQ}jW+UO?yG=&aPZ`u&)cNLS#c$E%;
zP1mx?uy^0lDm~1Sxl4AS3}fIGtsrpPzu9UsUFfLaHBjFbJbOC8(^<{SnS0n!StB#g
zL}+<5vF5jPm0~0Vjt%Q?>}z(+kvm<O%)1o_?8@mvGWjM3d^lAKD8n<$4a>w*edbY`
z@SqjXB|6rfvcCz`u_)Y;_+8ajHX`pJD&kNHczrrb!=?w{fiV124cV4iCdMz&zJwqK
zE{Xo+XoqLy?Bk=64cI=0e(aTjm0Xe#m-AMZ32X;}2a~{K(-z>-Z{9+k8X@v&$i>E+
zQXRR%1%i<O*1;l4ym4+YQ7kEYM+j&quO-W155Afghj5-RNC;+zFkN~j_%GksU3%O2
zUm#d|T|+M-vBUrB%H#fE*KJUimaYh$ahjF3XD=FhMXtQfGrqicbHWR2_G8@CPqCJ>
z+gZhDOYfLI95J{tK}^7qxH4qdT=iwIXI;%~<(7x#tDq?=+?_Bwkrz$Q`TdVlR))UZ
zB@j7mCy|Sc;&$|B7PbW~9eY=&!EdvD*IjQ)sB0(+cInMrs1{CN7*c3cOPQ|Rp{(Mt
zOjz!_eLXl-Mnq|<QCVMVaLfk;XD8dw=^}sjeJ-qHCviyrJzXpr!i4=;Qn;u`!h|W&
z#iP$WV+kX?YTqx$bUlm}$AOtqvH{!HHR|o1Lc!f81R}!Utl2s=Ke}tHI_fcY^MK#U
z&|exEn~O%q+R4vb3kB$S6ilN3xsd^SFJS@HEhgWkFCH}k<pN1RX3*nWy<#z=-IG3j
z68K@}m16OGxxw{8De@t@V$(_=E7QTGGJ|$A0jdHi^ccZ6`t#GaF?nrg1|ui;1DDhs
zrf<-%3`j<fi~C-!i@2wJA1%kZGccDd;9+n^Eoq%wpb*<gp?>AsoQfh;NJug*Q+l)T
z8)*rHeYv|M#Ut?%U?vsg>xas%hWwvYC#0-Sur)F2=FtkgqgCA#-e%z?+T?CPv=igi
zT1bbmLe@F14M@f`Vd)CEd&TVRZ9lERix`Pv&Rj93@;7`u^|14k-pNBn{fTijp&Qtp
z1B1q7E%*$Cjn_4DC1sJoqaH8jlQik)F@7bhGU9K_XgO4mt1xyV1wx@8p6R>!Z=zo}
zJP|z<nA5<}xSl1o8PzXm950@7@B>{6Si_^2=pvzAdCS%KC?wD74V_1_=k2eaRNll~
zg_)0a(KkYGbiWo&D8{^z3der9T|L|venrzv_U)qcQ^$MFo;M|<eiv0@D1H@2@$0u=
zz5}C}`<-(>wFp-P!qEO(#sfd<Umz|S4;SITWjr8(^{8Co{AoT2gMp}k4un7ePJ(n0
zG@e9k+Kb9r+?%T(_X+E@#kS=DU9^(H&$U?W<!?(dY8@Mi#~L2B_3Sl4_J=a{9A6_u
zgK79(cHtKF4(XbO1vxouU_S!JPL1Z$*QfQpRNqAnKK7Ork>Z1oL-r?nXuCiHy2Qq0
ziM;!sUs2F`&Az?Mh;}V|OcB@zA_b&cXvUK=;k<mB)QHU62*uekq~1H<+QIVUu<1<k
zg`t*mQ^5v<M&Fd`=A)bI$U_5F9B~Y!t(K8NT+^hH#s^mn0z`0~5}7O0nkp+9as4P3
zZ%lvDvKZODXkGtU)}{O^T8`Qx&qU|^Ae<ux0jn>^R`-C{MPC+tX)V!8h%5<NTsD56
zgPs%(Hw*UDW3~aRik^-tv7;eE0U$)Mx2kiaJQ2{3+#Y|vcyjrQfV-!mLUN&$EByO}
zU(9XQRQ7>RdT?@P(Uf7yaQA1qCnw{nH+*r>*BMBG9Ld(hllZ5Ey>&8FWz*Q!dPYoj
zCzFk0KZGQUrz|3zXT|9#%3tEMsddw5y@{5+yMt>KbeorB%SpCOL#uM0uGFAD<DZNj
z*<InzcC_AZj)Es5Y2>E&g#g#^|1;Itr9?;L0YErGBwTfC_I{Rc$*iOkJ|A4=@L50Z
x`UE0joFCFX(i+qD)1h^DcPJOtN>)kRbQgMUO*p!K@AKymmi%I+{Ip*({}0#og{%Mo

literal 0
HcmV?d00001

diff --git a/x-pack/qa/smoke-test-plugins-ssl/testnode.pem b/x-pack/qa/smoke-test-plugins-ssl/testnode.pem
new file mode 100644
index 00000000000..5a67e103344
--- /dev/null
+++ b/x-pack/qa/smoke-test-plugins-ssl/testnode.pem
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,9D867F7E0C94D013
+
+dVoVCjPeg1wgS7rVtOvGfQcrZyLkx393aWRnFq45tbjKBVuITtJ9vI7o4QXOV/15
+Gnb6WhXGIdWrzsxEAd46K6hIuNSISd4Emsx6c2Q5hTqWXXfexbOZBNfTtXtdJPnJ
+1jAaikhtztLo3JSLTKNY5sNxd+XbaQyYVUWvueK6zOaIIMETvB+VPVFd9i1ROibk
+Sgdtyj01KjkoalifqK/tA0CIYNKL0S6/eoK3UhAlpIprlpV+cnXa940C6bjLeJPt
+PMAGGp5RrplxSgrSerw3I9DOWkHGtpqzIka3XneNUXJP8k4HUJ+aZkGH2ZILKS8d
+4KMIb+KZSpHEGn+6uGccWLtZZmAjWJrDw56JbQtSHdRYLBRSOjLbTvQoPu/2Hpli
+7HOxbotlvjptMunncq5aqK57SHA1dh0cwF7J3LUmGFJ67eoz+VV3b5qMn4MopSeI
+mS16Ydd3nGpjSrln/elM0CQxqWfcOAXRZpDpFUQoXcBrLVzvz2DBl/0CrTRLhgzi
+CO+5/IVcBWRlYpRNGgjjP7q0j6URID3jk5J06fYQXmBiwQT5j+GZqqzpMCJ9mIy2
+1O9SN1hebJnIcEU+E0njn/MGjlYdPywhaCy8pqElp6Q8TUEJpwLRFO/owCoBet/n
+ZmCXUjfCGhc1pWHufFcDEQ6xMgEWWY/tdwCZeSU7EhErTjCbfupg+55A5fpDml0m
+3wH4CFcuRjlqyx6Ywixm1ATeitDtJl5HQTw6b8OtEXwSgRmZ0eSqSRVk9QbVS7gu
+IpQe09/Zimb5HzjZqZ3fdqHlcW4xax8hyJeyIvF5ZJ57eY8CBvu/wP2GDn26QnvF
+xQqdfDbq1H4JmpwUHpbFwBoQK4Q6WFd1z4EA9bRQeo3H9PoqoOwMDjzajwLRF7b7
+q6tYH/n9PyHwdf1c4fFwgSmL1toXGfKlA9hjIaLsRSDD6srT5EdUk78bsnddwI51
+tu7C7P4JG+h1VdRNMNTlqtileWsIE7Nn2A1OkcUxZdF5mamENpDpJcHePLto6c8q
+FKiwyFMsxhgsj6HK2HqO+UA4sX5Ni4oHwiPmb//EZLn045M5i1AN26KosJmb8++D
+sgR5reWRy+UqJCTYblVg+7Dx++ggUnfxVyQEsWmw5r5f4KU5wXBkvoVMGtPNa9DE
+n/uLtObD1qkNL38pRsr2OGRchYCgEoKGqEISBP4knfGXLOlWiW/246j9QzI97r1u
+tvy7fKg28G7AUz9l6bpewsPHefBUeRQeieP9eJINaEpxkF/w2RpKDLpQjWxwDDOM
+s+D0mrBMJve17AmJ8rMw6dIQPZYNZ88/jz1uQuUwQ2YlbmtZbCG81k9YMFGEU9XS
+cyhJxj8hvYnt2PR5Z9/cJPyWOs0m/ufOeeQQ8SnU/lzmrQnpzUd2Z6p5i/B7LdRP
+n1kX+l1qynuPnjvBz4nJQE0p6nzW8RyCDSniC9mtYtZmhgC8icqxgbvS7uEOBIYJ
+NbK+0bEETTO34iY/JVTIqLOw3iQZYMeUpxpj6Phgx/oooxMTquMecPKNgeVtaBst
+qjTNPX0ti1/HYpZqzYi8SV8YjHSJWCVMsZjKPr3W/HIcCKqYoIfgzi83Ha2KMQx6
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/qa/sql/security/ssl/build.gradle b/x-pack/qa/sql/security/ssl/build.gradle
index fe8aaeaff2b..cfc04f97188 100644
--- a/x-pack/qa/sql/security/ssl/build.gradle
+++ b/x-pack/qa/sql/security/ssl/build.gradle
@@ -1,5 +1,5 @@
+import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
-import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.NodeInfo
 
 import javax.net.ssl.HttpsURLConnection
@@ -22,7 +22,7 @@ Object san = new SanEvaluator()
 File keystoreDir = new File(project.buildDir, 'keystore')
 
 // Generate the node's keystore
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
+File nodeKeystore = file("$keystoreDir/test-node.jks")
 task createNodeKeyStore(type: LoggedExec) {
   doFirst {
     if (nodeKeystore.parentFile.exists() == false) {
@@ -47,7 +47,7 @@ task createNodeKeyStore(type: LoggedExec) {
 }
 
 // Generate the client's keystore
-File clientKeyStore = new File(keystoreDir, 'test-client.jks')
+File clientKeyStore = file("$keystoreDir/test-client.jks")
 task createClientKeyStore(type: LoggedExec) {
   doFirst {
     if (clientKeyStore.parentFile.exists() == false) {
@@ -72,7 +72,7 @@ task createClientKeyStore(type: LoggedExec) {
 }
 
 // Export the node's certificate
-File nodeCertificate = new File(keystoreDir, 'test-node.cert')
+File nodeCertificate = file("$keystoreDir/test-node.cert")
 task exportNodeCertificate(type: LoggedExec) {
   dependsOn createNodeKeyStore
   doFirst {
@@ -104,7 +104,7 @@ task importNodeCertificateInClientKeyStore(type: LoggedExec) {
 }
 
 // Export the client's certificate
-File clientCertificate = new File(keystoreDir, 'test-client.cert')
+File clientCertificate = file("$keystoreDir/test-client.cert")
 task exportClientCertificate(type: LoggedExec) {
   dependsOn createClientKeyStore
   doFirst {
@@ -145,7 +145,6 @@ processTestResources.dependsOn(importNodeCertificateInClientKeyStore, importClie
 
 integTestCluster.dependsOn(importClientCertificateInNodeKeyStore)
 
-
 integTestCluster {
   // The setup that we actually want
   setting 'xpack.security.http.ssl.enabled', 'true'
@@ -206,9 +205,18 @@ integTestCluster {
     return tmpFile.exists()
   }
 }
+Closure notRunningFips = {
+  Boolean.parseBoolean(BuildPlugin.runJavascript(project, project.runtimeJavaHome,
+          'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));')) == false
+}
 
-
-
+// Do not attempt to form a cluster in a FIPS JVM, as doing so with a JKS keystore will fail.
+// TODO Revisit this when SQL CLI client can handle key/certificate instead of only Keystores.
+// https://github.com/elastic/elasticsearch/issues/32306
+tasks.matching({ it.name == "integTestCluster#init" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "integTestCluster#start" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "integTestCluster#wait" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "integTestRunner" }).all { onlyIf notRunningFips }
 
 /** A lazy evaluator to find the san to use for certificate generation. */
 class SanEvaluator {
@@ -293,7 +301,7 @@ class SanEvaluator {
     byte[] bytes = inet6Address.getAddress();
     int[] hextets = new int[8];
     for (int i = 0; i < hextets.length; i++) {
-      hextets[i] =  (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255;
+      hextets[i] = (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255;
     }
     compressLongestRunOfZeroes(hextets);
     return hextetsToIPv6String(hextets);
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java
index a339222445a..301e15c8efb 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java
+++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java
@@ -84,7 +84,7 @@ public abstract class JdbcIntegrationTestCase extends ESRestTestCase {
     public static void index(String index, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
         index(index, "1", body);
     }
-    
+
     public static void index(String index, String documentId, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
         Request request = new Request("PUT", "/" + index + "/doc/" + documentId);
         request.addParameter("refresh", "true");

From 46709f1406a01e633d6805b40ed30b79b25ee47a Mon Sep 17 00:00:00 2001
From: Vladimir Dolzhenko <vladimir.dolzhenko@gmail.com>
Date: Tue, 24 Jul 2018 14:09:41 +0200
Subject: [PATCH 152/260] fixes broken build for third-party-tests (#32315)

fixes broken build for third-party-tests (#32315)
Relates #31918 / Closes infra/issues/6085
---
 plugins/repository-s3/build.gradle | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 181891e2056..13119913672 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -114,9 +114,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
 
   useFixture = true
 
-} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath
-        || !s3EC2Bucket || !s3EC2BasePath
-        || !s3ECSBucket || !s3ECSBasePath) {
+} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
   throw new IllegalArgumentException("not all options specified to run against external S3 service")
 }
 
@@ -349,8 +347,13 @@ processTestResources {
 
 project.afterEvaluate {
   if (useFixture == false) {
-    // 30_repository_temporary_credentials is not ready for CI yet
-    integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/30_repository_temporary_credentials/*'
+    // temporary_credentials, ec2_credentials and ecs_credentials are not ready for third-party-tests yet
+    integTestRunner.systemProperty 'tests.rest.blacklist',
+            [
+                'repository_s3/30_repository_temporary_credentials/*',
+                'repository_s3/40_repository_ec2_credentials/*',
+                'repository_s3/50_repository_ecs_credentials/*'
+            ].join(",")
   }
 }
 

From 59cf600e03f8148434655f2e2e23d88b52c20712 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Tue, 24 Jul 2018 16:05:43 +0200
Subject: [PATCH 153/260] Register ERR metric with NamedXContentRegistry
 (#32320)

This adds the ERR metric to the provided xContent parsers in the module and the
high level rest client registry. Also adding integration tests to make sure the
metric is correctly registered and usable from the client.
---
 .../org/elasticsearch/client/RankEvalIT.java  | 48 +++++++++++++++----
 .../client/RestHighLevelClientTests.java      | 10 ++--
 .../rankeval/ExpectedReciprocalRank.java      |  3 ++
 .../RankEvalNamedXContentProvider.java        |  5 ++
 .../index/rankeval/RankEvalPlugin.java        |  4 ++
 .../rest-api-spec/test/rank_eval/10_basic.yml | 34 +++++++++++++
 6 files changed, 92 insertions(+), 12 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
index 330afafd9ef..0af270cb051 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
@@ -22,7 +22,11 @@ package org.elasticsearch.client;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.rankeval.DiscountedCumulativeGain;
 import org.elasticsearch.index.rankeval.EvalQueryQuality;
+import org.elasticsearch.index.rankeval.EvaluationMetric;
+import org.elasticsearch.index.rankeval.ExpectedReciprocalRank;
+import org.elasticsearch.index.rankeval.MeanReciprocalRank;
 import org.elasticsearch.index.rankeval.PrecisionAtK;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.index.rankeval.RankEvalResponse;
@@ -35,8 +39,10 @@ import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -64,15 +70,7 @@ public class RankEvalIT extends ESRestHighLevelClientTestCase {
      * calculation where all unlabeled documents are treated as not relevant.
      */
     public void testRankEvalRequest() throws IOException {
-        SearchSourceBuilder testQuery = new SearchSourceBuilder();
-        testQuery.query(new MatchAllQueryBuilder());
-        List<RatedDocument> amsterdamRatedDocs = createRelevant("index" , "amsterdam1", "amsterdam2", "amsterdam3", "amsterdam4");
-        amsterdamRatedDocs.addAll(createRelevant("index2", "amsterdam0"));
-        RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", amsterdamRatedDocs, testQuery);
-        RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("index", "berlin"), testQuery);
-        List<RatedRequest> specifications = new ArrayList<>();
-        specifications.add(amsterdamRequest);
-        specifications.add(berlinRequest);
+        List<RatedRequest> specifications = createTestEvaluationSpec();
         PrecisionAtK metric = new PrecisionAtK(1, false, 10);
         RankEvalSpec spec = new RankEvalSpec(specifications, metric);
 
@@ -114,6 +112,38 @@ public class RankEvalIT extends ESRestHighLevelClientTestCase {
         response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
     }
 
+    private static List<RatedRequest> createTestEvaluationSpec() {
+        SearchSourceBuilder testQuery = new SearchSourceBuilder();
+        testQuery.query(new MatchAllQueryBuilder());
+        List<RatedDocument> amsterdamRatedDocs = createRelevant("index" , "amsterdam1", "amsterdam2", "amsterdam3", "amsterdam4");
+        amsterdamRatedDocs.addAll(createRelevant("index2", "amsterdam0"));
+        RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", amsterdamRatedDocs, testQuery);
+        RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("index", "berlin"), testQuery);
+        List<RatedRequest> specifications = new ArrayList<>();
+        specifications.add(amsterdamRequest);
+        specifications.add(berlinRequest);
+        return specifications;
+    }
+
+    /**
+     * Test case checks that the default metrics are registered and usable
+     */
+    public void testMetrics() throws IOException {
+        List<RatedRequest> specifications = createTestEvaluationSpec();
+        List<Supplier<EvaluationMetric>> metrics = Arrays.asList(PrecisionAtK::new, MeanReciprocalRank::new, DiscountedCumulativeGain::new,
+                () -> new ExpectedReciprocalRank(1));
+        double expectedScores[] = new double[] {0.4285714285714286, 0.75, 1.6408962261063627, 0.4407738095238095};
+        int i = 0;
+        for (Supplier<EvaluationMetric> metricSupplier : metrics) {
+            RankEvalSpec spec = new RankEvalSpec(specifications, metricSupplier.get());
+
+            RankEvalRequest rankEvalRequest = new RankEvalRequest(spec, new String[] { "index", "index2" });
+            RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
+            assertEquals(expectedScores[i], response.getMetricScore(), Double.MIN_VALUE);
+            i++;
+        }
+    }
+
     private static List<RatedDocument> createRelevant(String indexName, String... docs) {
         return Stream.of(docs).map(s -> new RatedDocument(indexName, s, 1)).collect(Collectors.toList());
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 64a344790ca..48934a9bed8 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
+
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
@@ -60,6 +61,7 @@ import org.elasticsearch.common.xcontent.cbor.CborXContent;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
 import org.elasticsearch.index.rankeval.DiscountedCumulativeGain;
 import org.elasticsearch.index.rankeval.EvaluationMetric;
+import org.elasticsearch.index.rankeval.ExpectedReciprocalRank;
 import org.elasticsearch.index.rankeval.MeanReciprocalRank;
 import org.elasticsearch.index.rankeval.MetricDetail;
 import org.elasticsearch.index.rankeval.PrecisionAtK;
@@ -616,7 +618,7 @@ public class RestHighLevelClientTests extends ESTestCase {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(8, namedXContents.size());
+        assertEquals(10, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -630,14 +632,16 @@ public class RestHighLevelClientTests extends ESTestCase {
         assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
-        assertEquals(Integer.valueOf(3), categories.get(EvaluationMetric.class));
+        assertEquals(Integer.valueOf(4), categories.get(EvaluationMetric.class));
         assertTrue(names.contains(PrecisionAtK.NAME));
         assertTrue(names.contains(DiscountedCumulativeGain.NAME));
         assertTrue(names.contains(MeanReciprocalRank.NAME));
-        assertEquals(Integer.valueOf(3), categories.get(MetricDetail.class));
+        assertTrue(names.contains(ExpectedReciprocalRank.NAME));
+        assertEquals(Integer.valueOf(4), categories.get(MetricDetail.class));
         assertTrue(names.contains(PrecisionAtK.NAME));
         assertTrue(names.contains(MeanReciprocalRank.NAME));
         assertTrue(names.contains(DiscountedCumulativeGain.NAME));
+        assertTrue(names.contains(ExpectedReciprocalRank.NAME));
     }
 
     public void testApiNamingConventions() throws Exception {
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java
index 4aac29f299d..39e1266504d 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java
@@ -65,6 +65,9 @@ public class ExpectedReciprocalRank implements EvaluationMetric {
 
     public static final String NAME = "expected_reciprocal_rank";
 
+    /**
+     * @param maxRelevance the highest expected relevance in the data
+     */
     public ExpectedReciprocalRank(int maxRelevance) {
         this(maxRelevance, null, DEFAULT_K);
     }
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java
index f2176113cdf..7eddcf9dff6 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java
@@ -37,12 +37,17 @@ public class RankEvalNamedXContentProvider implements NamedXContentProvider {
                 MeanReciprocalRank::fromXContent));
         namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(DiscountedCumulativeGain.NAME),
                 DiscountedCumulativeGain::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(ExpectedReciprocalRank.NAME),
+                ExpectedReciprocalRank::fromXContent));
+
         namedXContent.add(new NamedXContentRegistry.Entry(MetricDetail.class, new ParseField(PrecisionAtK.NAME),
                 PrecisionAtK.Detail::fromXContent));
         namedXContent.add(new NamedXContentRegistry.Entry(MetricDetail.class, new ParseField(MeanReciprocalRank.NAME),
                 MeanReciprocalRank.Detail::fromXContent));
         namedXContent.add(new NamedXContentRegistry.Entry(MetricDetail.class, new ParseField(DiscountedCumulativeGain.NAME),
                 DiscountedCumulativeGain.Detail::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(MetricDetail.class, new ParseField(ExpectedReciprocalRank.NAME),
+                ExpectedReciprocalRank.Detail::fromXContent));
         return namedXContent;
     }
 }
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
index 8ac2b7fbee5..0e5d754778f 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
@@ -60,10 +60,14 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin {
         namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new));
         namedWriteables.add(
                 new NamedWriteableRegistry.Entry(EvaluationMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(EvaluationMetric.class, ExpectedReciprocalRank.NAME, ExpectedReciprocalRank::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetail.class, PrecisionAtK.NAME, PrecisionAtK.Detail::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetail.class, MeanReciprocalRank.NAME, MeanReciprocalRank.Detail::new));
         namedWriteables.add(
                 new NamedWriteableRegistry.Entry(MetricDetail.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain.Detail::new));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(MetricDetail.class, ExpectedReciprocalRank.NAME, ExpectedReciprocalRank.Detail::new));
         return namedWriteables;
     }
 
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
index fe877b37a68..ebe23ae53f4 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
@@ -161,3 +161,37 @@ setup:
   - match: {details.berlin_query.metric_details.mean_reciprocal_rank: {"first_relevant": 2}}
   - match: {details.berlin_query.unrated_docs:  [ {"_index": "foo", "_id": "doc1"}]}
 
+---
+"Expected Reciprocal Rank":
+
+  - skip:
+      version: " - 6.3.99"
+      reason: ERR was introduced in 6.4
+
+  - do:
+      rank_eval:
+        body: {
+          "requests" : [
+            {
+                "id": "amsterdam_query",
+                "request": { "query": { "match" : {"text" : "amsterdam" }}},
+                "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
+            },
+            {
+                "id" : "berlin_query",
+                "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
+                "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
+            }
+          ],
+          "metric" : { 
+              "expected_reciprocal_rank": {
+                  "maximum_relevance" : 1,
+                  "k" : 5
+              }
+          }
+        }
+
+  - gt: {metric_score: 0.2083333}
+  - lt: {metric_score: 0.2083334}
+  - match: {details.amsterdam_query.metric_details.expected_reciprocal_rank.unrated_docs: 2}
+  - match: {details.berlin_query.metric_details.expected_reciprocal_rank.unrated_docs: 1}

From 73a38895fd8e6c38eea726686eb2fce26ee569d0 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@gmail.com>
Date: Tue, 24 Jul 2018 16:17:09 +0200
Subject: [PATCH 154/260] Add Restore Snapshot High Level REST API

With this commit we add the restore snapshot API to the Java high level
REST client.

Relates #27205
Relates #32155
---
 .../client/RequestConverters.java             |  15 ++
 .../elasticsearch/client/SnapshotClient.java  |  32 ++++
 .../client/RequestConvertersTests.java        |  26 ++++
 .../client/RestHighLevelClientTests.java      |   1 -
 .../org/elasticsearch/client/SnapshotIT.java  |  41 +++++
 .../SnapshotClientDocumentationIT.java        | 106 +++++++++++++
 .../snapshot/restore_snapshot.asciidoc        | 144 ++++++++++++++++++
 .../restore/RestoreSnapshotRequest.java       |  81 +++++++++-
 .../restore/RestoreSnapshotResponse.java      |  44 ++++++
 .../elasticsearch/snapshots/RestoreInfo.java  |  52 +++++--
 .../restore/RestoreSnapshotRequestTests.java  | 141 +++++++++++++++++
 .../restore/RestoreSnapshotResponseTests.java |  56 +++++++
 .../test/rest/ESRestTestCase.java             |   5 +
 13 files changed, 732 insertions(+), 12 deletions(-)
 create mode 100644 docs/java-rest/high-level/snapshot/restore_snapshot.asciidoc
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponseTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index a6122b0681e..b486d9fce52 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -40,6 +40,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
@@ -980,6 +981,20 @@ final class RequestConverters {
         return request;
     }
 
+    static Request restoreSnapshot(RestoreSnapshotRequest restoreSnapshotRequest) throws IOException {
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot")
+            .addPathPart(restoreSnapshotRequest.repository())
+            .addPathPart(restoreSnapshotRequest.snapshot())
+            .addPathPartAsIs("_restore")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        Params parameters = new Params(request);
+        parameters.withMasterTimeout(restoreSnapshotRequest.masterNodeTimeout());
+        parameters.withWaitForCompletion(restoreSnapshotRequest.waitForCompletion());
+        request.setEntity(createEntity(restoreSnapshotRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot")
             .addPathPart(deleteSnapshotRequest.repository())
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
index ae115839bae..319eb96a9f8 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java
@@ -30,6 +30,8 @@ import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyReposito
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
@@ -252,6 +254,36 @@ public final class SnapshotClient {
             SnapshotsStatusResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Restores a snapshot.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
+     * API on elastic.co</a>
+     *
+     * @param restoreSnapshotRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public RestoreSnapshotResponse restore(RestoreSnapshotRequest restoreSnapshotRequest, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(restoreSnapshotRequest, RequestConverters::restoreSnapshot, options,
+            RestoreSnapshotResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously restores a snapshot.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
+     * API on elastic.co</a>
+     *
+     * @param restoreSnapshotRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void restoreAsync(RestoreSnapshotRequest restoreSnapshotRequest, RequestOptions options,
+                            ActionListener<RestoreSnapshotResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(restoreSnapshotRequest, RequestConverters::restoreSnapshot, options,
+            RestoreSnapshotResponse::fromXContent, listener, emptySet());
+    }
+
     /**
      * Deletes a snapshot.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index c1f47feb33d..0415d363c54 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -41,6 +41,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequ
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
@@ -2198,6 +2199,31 @@ public class RequestConvertersTests extends ESTestCase {
         assertThat(request.getEntity(), is(nullValue()));
     }
 
+    public void testRestoreSnapshot() throws IOException {
+        Map<String, String> expectedParams = new HashMap<>();
+        String repository = randomIndicesNames(1, 1)[0];
+        String snapshot = "snapshot-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT);
+        String endpoint = String.format(Locale.ROOT, "/_snapshot/%s/%s/_restore", repository, snapshot);
+
+        RestoreSnapshotRequest restoreSnapshotRequest = new RestoreSnapshotRequest(repository, snapshot);
+        setRandomMasterTimeout(restoreSnapshotRequest, expectedParams);
+        if (randomBoolean()) {
+            restoreSnapshotRequest.waitForCompletion(true);
+            expectedParams.put("wait_for_completion", "true");
+        }
+        if (randomBoolean()) {
+            String timeout = randomTimeValue();
+            restoreSnapshotRequest.masterNodeTimeout(timeout);
+            expectedParams.put("master_timeout", timeout);
+        }
+
+        Request request = RequestConverters.restoreSnapshot(restoreSnapshotRequest);
+        assertThat(endpoint, equalTo(request.getEndpoint()));
+        assertThat(HttpPost.METHOD_NAME, equalTo(request.getMethod()));
+        assertThat(expectedParams, equalTo(request.getParameters()));
+        assertToXContentBody(restoreSnapshotRequest, request.getEntity());
+    }
+
     public void testDeleteSnapshot() {
         Map<String, String> expectedParams = new HashMap<>();
         String repository = randomIndicesNames(1, 1)[0];
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 48934a9bed8..5cf3b352756 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -665,7 +665,6 @@ public class RestHighLevelClientTests extends ESTestCase {
             "reindex_rethrottle",
             "render_search_template",
             "scripts_painless_execute",
-            "snapshot.restore",
             "tasks.get",
             "termvectors",
             "update_by_query"
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
index 5483f055c2c..06aec70a018 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
@@ -28,6 +28,8 @@ import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequ
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
 import org.elasticsearch.common.settings.Settings;
@@ -40,12 +42,15 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.snapshots.RestoreInfo;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 
 public class SnapshotIT extends ESRestHighLevelClientTestCase {
@@ -205,6 +210,42 @@ public class SnapshotIT extends ESRestHighLevelClientTestCase {
         assertThat(response.getSnapshots().get(0).getIndices().containsKey(testIndex), is(true));
     }
 
+    public void testRestoreSnapshot() throws IOException {
+        String testRepository = "test";
+        String testSnapshot = "snapshot_1";
+        String testIndex = "test_index";
+        String restoredIndex = testIndex + "_restored";
+
+        PutRepositoryResponse putRepositoryResponse = createTestRepository(testRepository, FsRepository.TYPE, "{\"location\": \".\"}");
+        assertTrue(putRepositoryResponse.isAcknowledged());
+
+        createIndex(testIndex, Settings.EMPTY);
+        assertTrue("index [" + testIndex + "] should have been created", indexExists(testIndex));
+
+        CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(testRepository, testSnapshot);
+        createSnapshotRequest.indices(testIndex);
+        createSnapshotRequest.waitForCompletion(true);
+        CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest);
+        assertEquals(RestStatus.OK, createSnapshotResponse.status());
+
+        deleteIndex(testIndex);
+        assertFalse("index [" + testIndex + "] should have been deleted", indexExists(testIndex));
+
+        RestoreSnapshotRequest request = new RestoreSnapshotRequest(testRepository, testSnapshot);
+        request.waitForCompletion(true);
+        request.renamePattern(testIndex);
+        request.renameReplacement(restoredIndex);
+
+        RestoreSnapshotResponse response = execute(request, highLevelClient().snapshot()::restore,
+            highLevelClient().snapshot()::restoreAsync);
+
+        RestoreInfo restoreInfo = response.getRestoreInfo();
+        assertThat(restoreInfo.name(), equalTo(testSnapshot));
+        assertThat(restoreInfo.indices(), equalTo(Collections.singletonList(restoredIndex)));
+        assertThat(restoreInfo.successfulShards(), greaterThan(0));
+        assertThat(restoreInfo.failedShards(), equalTo(0));
+    }
+
     public void testDeleteSnapshot() throws IOException {
         String repository = "test_repository";
         String snapshot = "test_snapshot";
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
index fff3e7ece70..922fcb984d9 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
@@ -33,6 +33,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotReq
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
@@ -53,12 +55,15 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 import org.elasticsearch.snapshots.SnapshotState;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -263,6 +268,107 @@ public class SnapshotClientDocumentationIT extends ESRestHighLevelClientTestCase
         }
     }
 
+    public void testRestoreSnapshot() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+
+        createTestRepositories();
+        createTestIndex();
+        createTestSnapshots();
+
+        // tag::restore-snapshot-request
+        RestoreSnapshotRequest request = new RestoreSnapshotRequest(repositoryName, snapshotName);
+        // end::restore-snapshot-request
+        // we need to restore as a different index name
+
+        // tag::restore-snapshot-request-masterTimeout
+        request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1>
+        request.masterNodeTimeout("1m"); // <2>
+        // end::restore-snapshot-request-masterTimeout
+
+        // tag::restore-snapshot-request-waitForCompletion
+        request.waitForCompletion(true); // <1>
+        // end::restore-snapshot-request-waitForCompletion
+
+        // tag::restore-snapshot-request-partial
+        request.partial(false); // <1>
+        // end::restore-snapshot-request-partial
+
+        // tag::restore-snapshot-request-include-global-state
+        request.includeGlobalState(false); // <1>
+        // end::restore-snapshot-request-include-global-state
+
+        // tag::restore-snapshot-request-include-aliases
+        request.includeAliases(false); // <1>
+        // end::restore-snapshot-request-include-aliases
+
+
+        // tag::restore-snapshot-request-indices
+        request.indices("test_index");
+        // end::restore-snapshot-request-indices
+
+        String restoredIndexName = "restored_index";
+        // tag::restore-snapshot-request-rename
+        request.renamePattern("test_(.+)"); // <1>
+        request.renameReplacement("restored_$1"); // <2>
+        // end::restore-snapshot-request-rename
+
+        // tag::restore-snapshot-request-index-settings
+        request.indexSettings(  // <1>
+            Settings.builder()
+            .put("index.number_of_replicas", 0)
+                .build());
+
+        request.ignoreIndexSettings("index.refresh_interval", "index.search.idle.after"); // <2>
+        request.indicesOptions(new IndicesOptions( // <3>
+            EnumSet.of(IndicesOptions.Option.IGNORE_UNAVAILABLE),
+            EnumSet.of(IndicesOptions.WildcardStates.OPEN)));
+        // end::restore-snapshot-request-index-settings
+
+        // tag::restore-snapshot-execute
+        RestoreSnapshotResponse response = client.snapshot().restore(request, RequestOptions.DEFAULT);
+        // end::restore-snapshot-execute
+
+        // tag::restore-snapshot-response
+        RestoreInfo restoreInfo = response.getRestoreInfo();
+        List<String> indices = restoreInfo.indices(); // <1>
+        // end::restore-snapshot-response
+        assertEquals(Collections.singletonList(restoredIndexName), indices);
+        assertEquals(0, restoreInfo.failedShards());
+        assertTrue(restoreInfo.successfulShards() > 0);
+    }
+
+    public void testRestoreSnapshotAsync() throws InterruptedException {
+        RestHighLevelClient client = highLevelClient();
+        {
+            RestoreSnapshotRequest request = new RestoreSnapshotRequest();
+
+            // tag::restore-snapshot-execute-listener
+            ActionListener<RestoreSnapshotResponse> listener =
+                new ActionListener<RestoreSnapshotResponse>() {
+                    @Override
+                    public void onResponse(RestoreSnapshotResponse restoreSnapshotResponse) {
+                        // <1>
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        // <2>
+                    }
+                };
+            // end::restore-snapshot-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::restore-snapshot-execute-async
+            client.snapshot().restoreAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::restore-snapshot-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
     public void testSnapshotDeleteRepository() throws IOException {
         RestHighLevelClient client = highLevelClient();
 
diff --git a/docs/java-rest/high-level/snapshot/restore_snapshot.asciidoc b/docs/java-rest/high-level/snapshot/restore_snapshot.asciidoc
new file mode 100644
index 00000000000..a4b83ca419a
--- /dev/null
+++ b/docs/java-rest/high-level/snapshot/restore_snapshot.asciidoc
@@ -0,0 +1,144 @@
+[[java-rest-high-snapshot-restore-snapshot]]
+=== Restore Snapshot API
+
+The Restore Snapshot API allows to restore a snapshot.
+
+[[java-rest-high-snapshot-restore-snapshot-request]]
+==== Restore Snapshot Request
+
+A `RestoreSnapshotRequest`:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request]
+--------------------------------------------------
+
+==== Limiting Indices to Restore
+
+By default all indices are restored. With the `indices` property you can
+provide a list of indices that should be restored:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request-indices]
+--------------------------------------------------
+<1> Request that Elasticsearch only restores "test_index".
+
+==== Renaming Indices
+
+You can rename indices using regular expressions when restoring a snapshot:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request-rename]
+--------------------------------------------------
+<1> A regular expression matching the indices that should be renamed.
+<2> A replacement pattern that references the group from the regular
+    expression as `$1`. "test_index" from the snapshot is restored as
+    "restored_index" in this example.
+
+==== Index Settings and Options
+
+You can also customize index settings and options when restoring:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request-index-settings]
+--------------------------------------------------
+<1> Use `#indexSettings()` to set any specific index setting for the indices
+    that are restored.
+<2> Use `#ignoreIndexSettings()` to provide index settings that should be
+    ignored from the original indices.
+<3> Set `IndicesOptions.Option.IGNORE_UNAVAILABLE` in `#indicesOptions()` to
+    have the restore succeed even if indices are missing in the snapshot.
+
+==== Further Arguments
+
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request-masterTimeout]
+--------------------------------------------------
+<1> Timeout to connect to the master node as a `TimeValue`
+<2> Timeout to connect to the master node as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request-waitForCompletion]
+--------------------------------------------------
+<1> Boolean indicating whether to wait until the snapshot has been restored.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request-partial]
+--------------------------------------------------
+<1> Boolean indicating whether the entire snapshot should succeed although one
+    or more indices participating in the snapshot don’t have all primary
+    shards available.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request-include-global-state]
+--------------------------------------------------
+<1> Boolean indicating whether restored templates that don’t currently exist
+    in the cluster are added and existing templates with the same name are
+    replaced by the restored templates. The restored persistent settings are
+    added to the existing persistent settings.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-request-include-aliases]
+--------------------------------------------------
+<1> Boolean to control whether aliases should be restored. Set to `false` to
+    prevent aliases from being restored together with associated indices.
+
+[[java-rest-high-snapshot-restore-snapshot-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-execute]
+--------------------------------------------------
+
+[[java-rest-high-snapshot-restore-snapshot-async]]
+==== Asynchronous Execution
+
+The asynchronous execution of a restore snapshot request requires both the
+`RestoreSnapshotRequest` instance and an `ActionListener` instance to be
+passed to the asynchronous method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-execute-async]
+--------------------------------------------------
+<1> The `RestoreSnapshotRequest` to execute and the `ActionListener`
+to use when the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `RestoreSnapshotResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+    provided as an argument.
+<2> Called in case of a failure. The raised exception is provided as an argument.
+
+[[java-rest-high-cluster-restore-snapshot-response]]
+==== Restore Snapshot Response
+
+The returned `RestoreSnapshotResponse` allows to retrieve information about the
+executed operation as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[restore-snapshot-response]
+--------------------------------------------------
+<1> The `RestoreInfo` contains details about the restored snapshot like the indices or
+    the number of successfully restored and failed shards.
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index c1b8c73c9ef..53aa522772a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -27,14 +27,17 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
@@ -45,7 +48,7 @@ import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBo
 /**
  * Restore snapshot request
  */
-public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotRequest> {
+public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotRequest> implements ToXContentObject {
 
     private String snapshot;
     private String repository;
@@ -563,6 +566,49 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
         return this;
     }
 
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startArray("indices");
+        for (String index : indices) {
+            builder.value(index);
+        }
+        builder.endArray();
+        if (indicesOptions != null) {
+            indicesOptions.toXContent(builder, params);
+        }
+        if (renamePattern != null) {
+            builder.field("rename_pattern", renamePattern);
+        }
+        if (renameReplacement != null) {
+            builder.field("rename_replacement", renameReplacement);
+        }
+        builder.field("include_global_state", includeGlobalState);
+        builder.field("partial", partial);
+        builder.field("include_aliases", includeAliases);
+        if (settings != null) {
+            builder.startObject("settings");
+            if (settings.isEmpty() == false) {
+                settings.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
+        if (indexSettings != null) {
+            builder.startObject("index_settings");
+            if (indexSettings.isEmpty() == false) {
+                indexSettings.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
+        builder.startArray("ignore_index_settings");
+        for (String ignoreIndexSetting : ignoreIndexSettings) {
+            builder.value(ignoreIndexSetting);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
@@ -573,4 +619,37 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
         return "snapshot [" + repository + ":" + snapshot + "]";
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RestoreSnapshotRequest that = (RestoreSnapshotRequest) o;
+        return waitForCompletion == that.waitForCompletion &&
+            includeGlobalState == that.includeGlobalState &&
+            partial == that.partial &&
+            includeAliases == that.includeAliases &&
+            Objects.equals(snapshot, that.snapshot) &&
+            Objects.equals(repository, that.repository) &&
+            Arrays.equals(indices, that.indices) &&
+            Objects.equals(indicesOptions, that.indicesOptions) &&
+            Objects.equals(renamePattern, that.renamePattern) &&
+            Objects.equals(renameReplacement, that.renameReplacement) &&
+            Objects.equals(settings, that.settings) &&
+            Objects.equals(indexSettings, that.indexSettings) &&
+            Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(snapshot, repository, indicesOptions, renamePattern, renameReplacement, waitForCompletion,
+            includeGlobalState, partial, includeAliases, settings, indexSettings);
+        result = 31 * result + Arrays.hashCode(indices);
+        result = 31 * result + Arrays.hashCode(ignoreIndexSettings);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
index 5a02e4bcb13..171509c0182 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
@@ -21,15 +21,21 @@ package org.elasticsearch.action.admin.cluster.snapshots.restore;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.RestoreInfo;
 
 import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
  * Contains information about restores snapshot
@@ -86,4 +92,42 @@ public class RestoreSnapshotResponse extends ActionResponse implements ToXConten
         builder.endObject();
         return builder;
     }
+
+    public static final ConstructingObjectParser<RestoreSnapshotResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "restore_snapshot", true, v -> {
+            RestoreInfo restoreInfo = (RestoreInfo) v[0];
+            Boolean accepted = (Boolean) v[1];
+            assert (accepted == null && restoreInfo != null) ||
+                   (accepted != null && accepted && restoreInfo == null) :
+                "accepted: [" + accepted + "], restoreInfo: [" + restoreInfo + "]";
+            return new RestoreSnapshotResponse(restoreInfo);
+    });
+
+    static {
+        PARSER.declareObject(optionalConstructorArg(), (parser, context) -> RestoreInfo.fromXContent(parser), new ParseField("snapshot"));
+        PARSER.declareBoolean(optionalConstructorArg(), new ParseField("accepted"));
+    }
+
+
+    public static RestoreSnapshotResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RestoreSnapshotResponse that = (RestoreSnapshotResponse) o;
+        return Objects.equals(restoreInfo, that.restoreInfo);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(restoreInfo);
+    }
+
+    @Override
+    public String toString() {
+        return "RestoreSnapshotResponse{" + "restoreInfo=" + restoreInfo + '}';
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java
index 36e80501fc1..6a58b52f72a 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreInfo.java
@@ -18,18 +18,22 @@
  */
 package org.elasticsearch.snapshots;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Information about successfully completed restore operation.
@@ -120,9 +124,6 @@ public class RestoreInfo implements ToXContentObject, Streamable {
         static final String SUCCESSFUL = "successful";
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -141,9 +142,23 @@ public class RestoreInfo implements ToXContentObject, Streamable {
         return builder;
     }
 
-    /**
-     * {@inheritDoc}
-     */
+    private static final ObjectParser<RestoreInfo, Void> PARSER = new ObjectParser<>(RestoreInfo.class.getName(), true, RestoreInfo::new);
+
+    static {
+        ObjectParser<RestoreInfo, Void> shardsParser = new ObjectParser<>("shards", true, null);
+        shardsParser.declareInt((r, s) -> r.totalShards = s, new ParseField(Fields.TOTAL));
+        shardsParser.declareInt((r, s) -> { /* only consume, don't set */ }, new ParseField(Fields.FAILED));
+        shardsParser.declareInt((r, s) -> r.successfulShards = s, new ParseField(Fields.SUCCESSFUL));
+
+        PARSER.declareString((r, n) -> r.name = n, new ParseField(Fields.SNAPSHOT));
+        PARSER.declareStringArray((r, i) -> r.indices = i, new ParseField(Fields.INDICES));
+        PARSER.declareField(shardsParser::parse, new ParseField(Fields.SHARDS), ObjectParser.ValueType.OBJECT);
+    }
+
+    public static RestoreInfo fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         name = in.readString();
@@ -157,9 +172,6 @@ public class RestoreInfo implements ToXContentObject, Streamable {
         successfulShards = in.readVInt();
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
@@ -193,4 +205,24 @@ public class RestoreInfo implements ToXContentObject, Streamable {
         return in.readOptionalStreamable(RestoreInfo::new);
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RestoreInfo that = (RestoreInfo) o;
+        return totalShards == that.totalShards &&
+            successfulShards == that.successfulShards &&
+            Objects.equals(name, that.name) &&
+            Objects.equals(indices, that.indices);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, indices, totalShards, successfulShards);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
new file mode 100644
index 00000000000..fbe8761a07d
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.snapshots.restore;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RestoreSnapshotRequestTests extends AbstractWireSerializingTestCase<RestoreSnapshotRequest> {
+    private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) {
+        if (randomBoolean()) {
+            List<String> indices = new ArrayList<>();
+            int count = randomInt(3) + 1;
+
+            for (int i = 0; i < count; ++i) {
+                indices.add(randomAlphaOfLength(randomInt(3) + 2));
+            }
+
+            instance.indices(indices);
+        }
+        if (randomBoolean()) {
+            instance.renamePattern(randomUnicodeOfLengthBetween(1, 100));
+        }
+        if (randomBoolean()) {
+            instance.renameReplacement(randomUnicodeOfLengthBetween(1, 100));
+        }
+        instance.partial(randomBoolean());
+        instance.includeAliases(randomBoolean());
+
+        if (randomBoolean()) {
+            Map<String, Object> settings = new HashMap<>();
+            int count = randomInt(3) + 1;
+
+            for (int i = 0; i < count; ++i) {
+                settings.put(randomAlphaOfLengthBetween(2, 5), randomAlphaOfLengthBetween(2, 5));
+            }
+
+            instance.settings(settings);
+        }
+        if (randomBoolean()) {
+            Map<String, Object> indexSettings = new HashMap<>();
+            int count = randomInt(3) + 1;
+
+            for (int i = 0; i < count; ++i) {
+                indexSettings.put(randomAlphaOfLengthBetween(2, 5), randomAlphaOfLengthBetween(2, 5));;
+            }
+            instance.indexSettings(indexSettings);
+        }
+
+        instance.includeGlobalState(randomBoolean());
+
+        if (randomBoolean()) {
+            Collection<IndicesOptions.WildcardStates> wildcardStates = randomSubsetOf(
+                Arrays.asList(IndicesOptions.WildcardStates.values()));
+            Collection<IndicesOptions.Option> options = randomSubsetOf(
+                Arrays.asList(IndicesOptions.Option.ALLOW_NO_INDICES, IndicesOptions.Option.IGNORE_UNAVAILABLE));
+
+            instance.indicesOptions(new IndicesOptions(
+                options.isEmpty() ? IndicesOptions.Option.NONE : EnumSet.copyOf(options),
+                wildcardStates.isEmpty() ? IndicesOptions.WildcardStates.NONE : EnumSet.copyOf(wildcardStates)));
+        }
+
+        instance.waitForCompletion(randomBoolean());
+
+        if (randomBoolean()) {
+            instance.masterNodeTimeout(randomTimeValue());
+        }
+        return instance;
+    }
+
+    @Override
+    protected RestoreSnapshotRequest createTestInstance() {
+        return randomState(new RestoreSnapshotRequest(randomAlphaOfLength(5), randomAlphaOfLength(10)));
+    }
+
+    @Override
+    protected Writeable.Reader<RestoreSnapshotRequest> instanceReader() {
+        return RestoreSnapshotRequest::new;
+    }
+
+    @Override
+    protected RestoreSnapshotRequest mutateInstance(RestoreSnapshotRequest instance) throws IOException {
+        RestoreSnapshotRequest copy = copyInstance(instance);
+        // ensure that at least one property is different
+        copy.repository("copied-" + instance.repository());
+        return randomState(copy);
+    }
+
+    public void testSource() throws IOException {
+        RestoreSnapshotRequest original = createTestInstance();
+        XContentBuilder builder = original.toXContent(XContentFactory.jsonBuilder(), new ToXContent.MapParams(Collections.emptyMap()));
+        XContentParser parser = XContentType.JSON.xContent().createParser(
+            NamedXContentRegistry.EMPTY, null, BytesReference.bytes(builder).streamInput());
+        Map<String, Object> map = parser.mapOrdered();
+
+        // we will only restore properties from the map that are contained in the request body. All other
+        // properties are restored from the original (in the actual REST action this is restored from the
+        // REST path and request parameters).
+        RestoreSnapshotRequest processed = new RestoreSnapshotRequest(original.repository(), original.snapshot());
+        processed.masterNodeTimeout(original.masterNodeTimeout());
+        processed.waitForCompletion(original.waitForCompletion());
+
+        processed.source(map);
+
+        assertEquals(original, processed);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponseTests.java
new file mode 100644
index 00000000000..17d1ecafabd
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponseTests.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.snapshots.restore;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.snapshots.RestoreInfo;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class RestoreSnapshotResponseTests extends AbstractXContentTestCase<RestoreSnapshotResponse> {
+
+    @Override
+    protected RestoreSnapshotResponse createTestInstance() {
+        if (randomBoolean()) {
+            String name = randomRealisticUnicodeOfCodepointLengthBetween(1, 30);
+            List<String> indices = new ArrayList<>();
+            indices.add("test0");
+            indices.add("test1");
+            int totalShards = randomIntBetween(1, 1000);
+            int successfulShards = randomIntBetween(0, totalShards);
+            return new RestoreSnapshotResponse(new RestoreInfo(name, indices, totalShards, successfulShards));
+        } else {
+            return new RestoreSnapshotResponse(null);
+        }
+    }
+
+    @Override
+    protected RestoreSnapshotResponse doParseInstance(XContentParser parser) throws IOException {
+        return RestoreSnapshotResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 937adddf3a4..e25e8b43d3a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -536,6 +536,11 @@ public abstract class ESRestTestCase extends ESTestCase {
         client().performRequest(request);
     }
 
+    protected static void deleteIndex(String name) throws IOException {
+        Request request = new Request("DELETE", "/" + name);
+        client().performRequest(request);
+    }
+
     protected static void updateIndexSettings(String index, Settings.Builder settings) throws IOException {
         updateIndexSettings(index, settings.build());
     }

From 99426eb4f81a4a89f2434afc21821093c6c28b1c Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Tue, 24 Jul 2018 15:22:57 +0100
Subject: [PATCH 155/260] [ML] Extract persistent task methods from MlMetadata
 (#32319)

Move ML persistent task helper functions to the new class MlTasks
and remove MLMetadataField after moving the string constant to
MlMetadata.
---
 .../elasticsearch/xpack/core/XPackPlugin.java |  4 +-
 .../xpack/core/ml/MLMetadataField.java        | 21 ------
 .../xpack/core/ml/MlMetadata.java             | 59 ++--------------
 .../elasticsearch/xpack/core/ml/MlTasks.java  | 70 +++++++++++++++++++
 .../core/ml/action/IsolateDatafeedAction.java |  9 +--
 .../core/ml/action/StopDatafeedAction.java    |  4 +-
 .../ml/action/TransportCloseJobAction.java    | 13 ++--
 .../action/TransportDeleteDatafeedAction.java |  6 +-
 .../ml/action/TransportDeleteJobAction.java   |  6 +-
 .../TransportFinalizeJobExecutionAction.java  |  3 +-
 .../TransportGetDatafeedsStatsAction.java     |  5 +-
 .../action/TransportGetJobsStatsAction.java   |  9 +--
 .../TransportIsolateDatafeedAction.java       |  4 +-
 .../ml/action/TransportJobTaskAction.java     |  4 +-
 .../ml/action/TransportKillProcessAction.java |  4 +-
 .../ml/action/TransportOpenJobAction.java     |  3 +-
 .../ml/action/TransportPersistJobAction.java  |  4 +-
 .../ml/action/TransportPutDatafeedAction.java |  3 +-
 .../action/TransportStartDatafeedAction.java  |  6 +-
 .../action/TransportStopDatafeedAction.java   | 12 ++--
 .../action/TransportUpdateDatafeedAction.java |  3 +-
 .../xpack/ml/datafeed/DatafeedManager.java    |  5 +-
 .../ml/datafeed/DatafeedNodeSelector.java     |  3 +-
 .../xpack/ml/job/JobManager.java              |  8 +--
 .../ml/MachineLearningFeatureSetTests.java    |  3 +-
 .../ml/MlInitializationServiceTests.java      |  3 +-
 .../xpack/ml/MlMetadataTests.java             | 21 +-----
 .../elasticsearch/xpack/ml/MlTasksTests.java  | 68 ++++++++++++++++++
 .../action/TransportCloseJobActionTests.java  | 20 +++---
 .../action/TransportDeleteJobActionTests.java |  5 +-
 .../action/TransportOpenJobActionTests.java   | 14 ++--
 .../TransportStopDatafeedActionTests.java     | 10 +--
 .../ml/datafeed/DatafeedManagerTests.java     | 17 +++--
 .../datafeed/DatafeedNodeSelectorTests.java   |  6 +-
 .../integration/BasicDistributedJobsIT.java   |  6 +-
 .../xpack/ml/integration/DeleteJobIT.java     |  5 +-
 .../xpack/ml/integration/TooManyJobsIT.java   |  6 +-
 .../xpack/ml/job/JobManagerTests.java         | 17 +++--
 .../ExpiredModelSnapshotsRemoverTests.java    |  3 +-
 .../retention/ExpiredResultsRemoverTests.java |  3 +-
 40 files changed, 262 insertions(+), 213 deletions(-)
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MLMetadataField.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
index d14c72383d6..aaa3effcfe8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
@@ -52,7 +52,7 @@ import org.elasticsearch.xpack.core.action.TransportXPackInfoAction;
 import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
+import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction;
 import org.elasticsearch.xpack.core.rest.action.RestXPackUsageAction;
 import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
@@ -197,7 +197,7 @@ public class XPackPlugin extends XPackClientPlugin implements ScriptPlugin, Exte
     private static boolean alreadyContainsXPackCustomMetadata(ClusterState clusterState) {
         final MetaData metaData = clusterState.metaData();
         return metaData.custom(LicensesMetaData.TYPE) != null ||
-            metaData.custom(MLMetadataField.TYPE) != null ||
+            metaData.custom(MlMetadata.TYPE) != null ||
             metaData.custom(WatcherMetaData.TYPE) != null ||
             clusterState.custom(TokenMetaData.TYPE) != null;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MLMetadataField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MLMetadataField.java
deleted file mode 100644
index bef1b57902d..00000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MLMetadataField.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.ml;
-
-public final class MLMetadataField {
-
-    public static final String TYPE = "ml";
-
-    private MLMetadataField() {}
-
-    /**
-     * Namespaces the task ids for datafeeds.
-     * A job id can be used as a datafeed id, because they are stored separately in cluster state.
-     */
-    public static String datafeedTaskId(String datafeedId) {
-        return "datafeed-" + datafeedId;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
index 85e5c99fe35..f3505835994 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
@@ -55,6 +55,7 @@ import java.util.stream.Collectors;
 
 public class MlMetadata implements XPackPlugin.XPackMetaDataCustom {
 
+    public static final String TYPE = "ml";
     private static final ParseField JOBS_FIELD = new ParseField("jobs");
     private static final ParseField DATAFEEDS_FIELD = new ParseField("datafeeds");
 
@@ -119,7 +120,7 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom {
 
     @Override
     public String getWriteableName() {
-        return MLMetadataField.TYPE;
+        return TYPE;
     }
 
     @Override
@@ -213,7 +214,7 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom {
 
         @Override
         public String getWriteableName() {
-            return MLMetadataField.TYPE;
+            return TYPE;
         }
 
         static Diff<Job> readJobDiffFrom(StreamInput in) throws IOException {
@@ -277,7 +278,7 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom {
         public Builder deleteJob(String jobId, PersistentTasksCustomMetaData tasks) {
             checkJobHasNoDatafeed(jobId);
 
-            JobState jobState = MlMetadata.getJobState(jobId, tasks);
+            JobState jobState = MlTasks.getJobState(jobId, tasks);
             if (jobState.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) {
                 throw ExceptionsHelper.conflictStatusException("Unexpected job state [" + jobState + "], expected [" +
                         JobState.CLOSED + " or " + JobState.FAILED + "]");
@@ -362,7 +363,7 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom {
 
         private void checkDatafeedIsStopped(Supplier<String> msg, String datafeedId, PersistentTasksCustomMetaData persistentTasks) {
             if (persistentTasks != null) {
-                if (persistentTasks.getTask(MLMetadataField.datafeedTaskId(datafeedId)) != null) {
+                if (persistentTasks.getTask(MlTasks.datafeedTaskId(datafeedId)) != null) {
                     throw ExceptionsHelper.conflictStatusException(msg.get());
                 }
             }
@@ -399,7 +400,7 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom {
             checkJobHasNoDatafeed(jobId);
 
             if (allowDeleteOpenJob == false) {
-                PersistentTask<?> jobTask = getJobTask(jobId, tasks);
+                PersistentTask<?> jobTask = MlTasks.getJobTask(jobId, tasks);
                 if (jobTask != null) {
                     JobTaskState jobTaskState = (JobTaskState) jobTask.getState();
                     throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] because the job is "
@@ -420,56 +421,10 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom {
         }
     }
 
-    /**
-     * Namespaces the task ids for jobs.
-     * A datafeed id can be used as a job id, because they are stored separately in cluster state.
-     */
-    public static String jobTaskId(String jobId) {
-        return "job-" + jobId;
-    }
 
-    @Nullable
-    public static PersistentTask<?> getJobTask(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
-        if (tasks == null) {
-            return null;
-        }
-        return tasks.getTask(jobTaskId(jobId));
-    }
-
-    @Nullable
-    public static PersistentTask<?> getDatafeedTask(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
-        if (tasks == null) {
-            return null;
-        }
-        return tasks.getTask(MLMetadataField.datafeedTaskId(datafeedId));
-    }
-
-    public static JobState getJobState(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
-        PersistentTask<?> task = getJobTask(jobId, tasks);
-        if (task != null) {
-            JobTaskState jobTaskState = (JobTaskState) task.getState();
-            if (jobTaskState == null) {
-                return JobState.OPENING;
-            }
-            return jobTaskState.getState();
-        }
-        // If we haven't opened a job than there will be no persistent task, which is the same as if the job was closed
-        return JobState.CLOSED;
-    }
-
-    public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
-        PersistentTask<?> task = getDatafeedTask(datafeedId, tasks);
-        if (task != null && task.getState() != null) {
-            return (DatafeedState) task.getState();
-        } else {
-            // If we haven't started a datafeed then there will be no persistent task,
-            // which is the same as if the datafeed was't started
-            return DatafeedState.STOPPED;
-        }
-    }
 
     public static MlMetadata getMlMetadata(ClusterState state) {
-        MlMetadata mlMetadata = (state == null) ? null : state.getMetaData().custom(MLMetadataField.TYPE);
+        MlMetadata mlMetadata = (state == null) ? null : state.getMetaData().custom(TYPE);
         if (mlMetadata == null) {
             return EMPTY_METADATA;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
new file mode 100644
index 00000000000..5c17271738e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.ml;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
+import org.elasticsearch.xpack.core.ml.job.config.JobState;
+import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
+
+public final class MlTasks {
+
+    private MlTasks() {
+    }
+
+    /**
+     * Namespaces the task ids for jobs.
+     * A datafeed id can be used as a job id, because they are stored separately in cluster state.
+     */
+    public static String jobTaskId(String jobId) {
+        return "job-" + jobId;
+    }
+
+    /**
+     * Namespaces the task ids for datafeeds.
+     * A job id can be used as a datafeed id, because they are stored separately in cluster state.
+     */
+    public static String datafeedTaskId(String datafeedId) {
+        return "datafeed-" + datafeedId;
+    }
+
+    @Nullable
+    public static PersistentTasksCustomMetaData.PersistentTask<?> getJobTask(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
+        return tasks == null ? null : tasks.getTask(jobTaskId(jobId));
+    }
+
+    @Nullable
+    public static PersistentTasksCustomMetaData.PersistentTask<?> getDatafeedTask(String datafeedId,
+                                                                                  @Nullable PersistentTasksCustomMetaData tasks) {
+        return tasks == null ? null : tasks.getTask(datafeedTaskId(datafeedId));
+    }
+
+    public static JobState getJobState(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
+        PersistentTasksCustomMetaData.PersistentTask<?> task = getJobTask(jobId, tasks);
+        if (task != null) {
+            JobTaskState jobTaskState = (JobTaskState) task.getState();
+            if (jobTaskState == null) {
+                return JobState.OPENING;
+            }
+            return jobTaskState.getState();
+        }
+        // If we haven't opened a job than there will be no persistent task, which is the same as if the job was closed
+        return JobState.CLOSED;
+    }
+
+    public static DatafeedState getDatafeedState(String datafeedId, @Nullable PersistentTasksCustomMetaData tasks) {
+        PersistentTasksCustomMetaData.PersistentTask<?> task = getDatafeedTask(datafeedId, tasks);
+        if (task != null && task.getState() != null) {
+            return (DatafeedState) task.getState();
+        } else {
+            // If we haven't started a datafeed then there will be no persistent task,
+            // which is the same as if the datafeed was't started
+            return DatafeedState.STOPPED;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/IsolateDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/IsolateDatafeedAction.java
index 451679f3646..48cef12f01c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/IsolateDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/IsolateDatafeedAction.java
@@ -19,7 +19,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
@@ -84,11 +84,8 @@ public class IsolateDatafeedAction extends Action<IsolateDatafeedAction.Response
 
         @Override
         public boolean match(Task task) {
-            String expectedDescription = MLMetadataField.datafeedTaskId(datafeedId);
-            if (task instanceof StartDatafeedAction.DatafeedTaskMatcher && expectedDescription.equals(task.getDescription())){
-                return true;
-            }
-            return false;
+            String expectedDescription = MlTasks.datafeedTaskId(datafeedId);
+            return task instanceof StartDatafeedAction.DatafeedTaskMatcher && expectedDescription.equals(task.getDescription());
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
index 01172251410..c802f0bccca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
@@ -22,7 +22,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
@@ -125,7 +125,7 @@ public class StopDatafeedAction extends Action<StopDatafeedAction.Response> {
         @Override
         public boolean match(Task task) {
             for (String id : resolvedStartedDatafeedIds) {
-                String expectedDescription = MLMetadataField.datafeedTaskId(id);
+                String expectedDescription = MlTasks.datafeedTaskId(id);
                 if (task instanceof StartDatafeedAction.DatafeedTaskMatcher && expectedDescription.equals(task.getDescription())){
                     return true;
                 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
index 05810b943be..6fa47d1e459 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
@@ -27,6 +27,7 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -121,7 +122,7 @@ public class TransportCloseJobAction extends TransportTasksAction<TransportOpenJ
     private static void addJobAccordingToState(String jobId, PersistentTasksCustomMetaData tasksMetaData,
                                                List<String> openJobs, List<String> closingJobs, List<String> failedJobs) {
 
-        JobState jobState = MlMetadata.getJobState(jobId, tasksMetaData);
+        JobState jobState = MlTasks.getJobState(jobId, tasksMetaData);
         switch (jobState) {
             case CLOSING:
                 closingJobs.add(jobId);
@@ -143,7 +144,7 @@ public class TransportCloseJobAction extends TransportTasksAction<TransportOpenJ
         TransportCloseJobAction.WaitForCloseRequest waitForCloseRequest = new TransportCloseJobAction.WaitForCloseRequest();
 
         for (String jobId : openJobIds) {
-            PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks);
+            PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlTasks.getJobTask(jobId, tasks);
             if (jobTask != null) {
                 auditor.info(jobId, Messages.JOB_AUDIT_CLOSING);
                 waitForCloseRequest.persistentTaskIds.add(jobTask.getId());
@@ -151,7 +152,7 @@ public class TransportCloseJobAction extends TransportTasksAction<TransportOpenJ
             }
         }
         for (String jobId : closingJobIds) {
-            PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks);
+            PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlTasks.getJobTask(jobId, tasks);
             if (jobTask != null) {
                 waitForCloseRequest.persistentTaskIds.add(jobTask.getId());
             }
@@ -180,7 +181,7 @@ public class TransportCloseJobAction extends TransportTasksAction<TransportOpenJ
 
         Optional<DatafeedConfig> datafeed = mlMetadata.getDatafeedByJobId(jobId);
         if (datafeed.isPresent()) {
-            DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeed.get().getId(), tasks);
+            DatafeedState datafeedState = MlTasks.getDatafeedState(datafeed.get().getId(), tasks);
             if (datafeedState != DatafeedState.STOPPED) {
                 throw ExceptionsHelper.conflictStatusException("cannot close job [{}], datafeed hasn't been stopped", jobId);
             }
@@ -230,7 +231,7 @@ public class TransportCloseJobAction extends TransportTasksAction<TransportOpenJ
                 Set<String> executorNodes = new HashSet<>();
                 PersistentTasksCustomMetaData tasks = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
                 for (String resolvedJobId : request.getOpenJobIds()) {
-                    PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(resolvedJobId, tasks);
+                    PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlTasks.getJobTask(resolvedJobId, tasks);
                     if (jobTask == null || jobTask.isAssigned() == false) {
                         String message = "Cannot close job [" + resolvedJobId + "] because the job does not have an assigned node." +
                                 " Use force close to close the job";
@@ -312,7 +313,7 @@ public class TransportCloseJobAction extends TransportTasksAction<TransportOpenJ
         final AtomicArray<Exception> failures = new AtomicArray<>(numberOfJobs);
 
         for (String jobId : jobIdsToForceClose) {
-            PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks);
+            PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlTasks.getJobTask(jobId, tasks);
             if (jobTask != null) {
                 auditor.info(jobId, Messages.JOB_AUDIT_FORCE_CLOSING);
                 persistentTasksService.sendRemoveRequest(jobTask.getId(),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
index be7ac84284d..2582cb0082e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
@@ -22,8 +22,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackPlugin;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.IsolateDatafeedAction;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
@@ -86,7 +86,7 @@ public class TransportDeleteDatafeedAction extends TransportMasterNodeAction<Del
 
     private void removeDatafeedTask(DeleteDatafeedAction.Request request, ClusterState state, ActionListener<Boolean> listener) {
         PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        PersistentTasksCustomMetaData.PersistentTask<?> datafeedTask = MlMetadata.getDatafeedTask(request.getDatafeedId(), tasks);
+        PersistentTasksCustomMetaData.PersistentTask<?> datafeedTask = MlTasks.getDatafeedTask(request.getDatafeedId(), tasks);
         if (datafeedTask == null) {
             listener.onResponse(true);
         } else {
@@ -128,7 +128,7 @@ public class TransportDeleteDatafeedAction extends TransportMasterNodeAction<Del
                         MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata)
                                 .removeDatafeed(request.getDatafeedId(), persistentTasks).build();
                         return ClusterState.builder(currentState).metaData(
-                                MetaData.builder(currentState.getMetaData()).putCustom(MLMetadataField.TYPE, newMetadata).build())
+                                MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build())
                                 .build();
                     }
                 });
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
index ede92fbbab9..ad0d4068592 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
@@ -26,9 +26,9 @@ import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
 import org.elasticsearch.xpack.core.ml.action.KillProcessAction;
 import org.elasticsearch.xpack.core.ml.job.persistence.JobStorageDeletionTask;
@@ -177,7 +177,7 @@ public class TransportDeleteJobAction extends TransportMasterNodeAction<DeleteJo
                                       ActionListener<Boolean> listener) {
         PersistentTasksCustomMetaData tasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
 
-        PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks);
+        PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlTasks.getJobTask(jobId, tasks);
         if (jobTask == null) {
             listener.onResponse(null);
         } else {
@@ -251,7 +251,7 @@ public class TransportDeleteJobAction extends TransportMasterNodeAction<DeleteJo
 
     private static ClusterState buildNewClusterState(ClusterState currentState, MlMetadata.Builder builder) {
         ClusterState.Builder newState = ClusterState.builder(currentState);
-        newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MLMetadataField.TYPE, builder.build()).build());
+        newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, builder.build()).build());
         return newState.build();
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
index 8e7a0fef41e..ff611f92202 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
@@ -21,7 +21,6 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 
@@ -70,7 +69,7 @@ public class TransportFinalizeJobExecutionAction extends TransportMasterNodeActi
                 }
                 ClusterState.Builder builder = ClusterState.builder(currentState);
                 return builder.metaData(new MetaData.Builder(currentState.metaData())
-                        .putCustom(MLMetadataField.TYPE, mlMetadataBuilder.build()))
+                        .putCustom(MlMetadata.TYPE, mlMetadataBuilder.build()))
                         .build();
             }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
index 41c8379c39f..1f7e55fc488 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
@@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -69,8 +70,8 @@ public class TransportGetDatafeedsStatsAction extends TransportMasterNodeReadAct
 
     private static GetDatafeedsStatsAction.Response.DatafeedStats getDatafeedStats(String datafeedId, ClusterState state,
                                                                                    PersistentTasksCustomMetaData tasks) {
-        PersistentTasksCustomMetaData.PersistentTask<?> task = MlMetadata.getDatafeedTask(datafeedId, tasks);
-        DatafeedState datafeedState = MlMetadata.getDatafeedState(datafeedId, tasks);
+        PersistentTasksCustomMetaData.PersistentTask<?> task = MlTasks.getDatafeedTask(datafeedId, tasks);
+        DatafeedState datafeedState = MlTasks.getDatafeedState(datafeedId, tasks);
         DiscoveryNode node = null;
         String explanation = null;
         if (task != null) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
index 31f918dfc25..28034d757da 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
@@ -23,6 +23,7 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
@@ -102,9 +103,9 @@ public class TransportGetJobsStatsAction extends TransportTasksAction<TransportO
         PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
         Optional<Tuple<DataCounts, ModelSizeStats>> stats = processManager.getStatistics(task);
         if (stats.isPresent()) {
-            PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks);
+            PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlTasks.getJobTask(jobId, tasks);
             DiscoveryNode node = state.nodes().get(pTask.getExecutorNode());
-            JobState jobState = MlMetadata.getJobState(jobId, tasks);
+            JobState jobState = MlTasks.getJobState(jobId, tasks);
             String assignmentExplanation = pTask.getAssignment().getExplanation();
             TimeValue openTime = durationToTimeValue(processManager.jobOpenTime(task));
             gatherForecastStats(jobId, forecastStats -> {
@@ -137,8 +138,8 @@ public class TransportGetJobsStatsAction extends TransportTasksAction<TransportO
             String jobId = jobIds.get(i);
             gatherForecastStats(jobId, forecastStats -> {
                 gatherDataCountsAndModelSizeStats(jobId, (dataCounts, modelSizeStats) -> {
-                    JobState jobState = MlMetadata.getJobState(jobId, tasks);
-                    PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlMetadata.getJobTask(jobId, tasks);
+                    JobState jobState = MlTasks.getJobState(jobId, tasks);
+                    PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlTasks.getJobTask(jobId, tasks);
                     String assignmentExplanation = null;
                     if (pTask != null) {
                         assignmentExplanation = pTask.getAssignment().getExplanation();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
index 398a1007ff9..3ca3c315450 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
@@ -21,7 +21,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.IsolateDatafeedAction;
 import org.elasticsearch.xpack.ml.MachineLearning;
 
@@ -43,7 +43,7 @@ public class TransportIsolateDatafeedAction extends TransportTasksAction<Transpo
     protected void doExecute(Task task, IsolateDatafeedAction.Request request, ActionListener<IsolateDatafeedAction.Response> listener) {
         final ClusterState state = clusterService.state();
         PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        PersistentTasksCustomMetaData.PersistentTask<?> datafeedTask = MlMetadata.getDatafeedTask(request.getDatafeedId(), tasks);
+        PersistentTasksCustomMetaData.PersistentTask<?> datafeedTask = MlTasks.getDatafeedTask(request.getDatafeedId(), tasks);
 
         if (datafeedTask == null || datafeedTask.getExecutorNode() == null) {
             // No running datafeed task to isolate
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
index bd489588da3..07bb6152e8c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
@@ -18,7 +18,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.JobTaskRequest;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.job.JobManager;
@@ -55,7 +55,7 @@ public abstract class TransportJobTaskAction<Request extends JobTaskRequest<Requ
         ClusterState state = clusterService.state();
         JobManager.getJobOrThrowIfUnknown(jobId, state);
         PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(jobId, tasks);
+        PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlTasks.getJobTask(jobId, tasks);
         if (jobTask == null || jobTask.isAssigned() == false) {
             String message = "Cannot perform requested action because job [" + jobId + "] is not open";
             listener.onFailure(ExceptionsHelper.conflictStatusException(message));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
index 40cec95fae2..b40f0368a15 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
@@ -18,7 +18,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.KillProcessAction;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -59,7 +59,7 @@ public class TransportKillProcessAction extends TransportJobTaskAction<KillProce
     protected void doExecute(Task task, KillProcessAction.Request request, ActionListener<KillProcessAction.Response> listener) {
         DiscoveryNodes nodes = clusterService.state().nodes();
         PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(request.getJobId(), tasks);
+        PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlTasks.getJobTask(request.getJobId(), tasks);
         if (jobTask == null || jobTask.getExecutorNode() == null) {
             logger.debug("[{}] Cannot kill the process because job is not open", request.getJobId());
             listener.onResponse(new KillProcessAction.Response(false));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
index e82e53458cb..d378b19aad5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
@@ -51,6 +51,7 @@ import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateJobAction;
@@ -489,7 +490,7 @@ public class TransportOpenJobAction extends TransportMasterNodeAction<OpenJobAct
 
             // Step 4. Start job task
             ActionListener<PutJobAction.Response> establishedMemoryUpdateListener = ActionListener.wrap(
-                    response -> persistentTasksService.sendStartRequest(MlMetadata.jobTaskId(jobParams.getJobId()),
+                    response -> persistentTasksService.sendStartRequest(MlTasks.jobTaskId(jobParams.getJobId()),
                             OpenJobAction.TASK_NAME, jobParams, finalListener),
                     listener::onFailure
             );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java
index 926395d6513..1b90d5cb04b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java
@@ -19,7 +19,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.PersistJobAction;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
@@ -61,7 +61,7 @@ public class TransportPersistJobAction extends TransportJobTaskAction<PersistJob
         // TODO Remove this overridden method in 7.0.0
         DiscoveryNodes nodes = clusterService.state().nodes();
         PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlMetadata.getJobTask(request.getJobId(), tasks);
+        PersistentTasksCustomMetaData.PersistentTask<?> jobTask = MlTasks.getJobTask(request.getJobId(), tasks);
         if (jobTask == null || jobTask.getExecutorNode() == null) {
             logger.debug("[{}] Cannot persist the job because the job is not open", request.getJobId());
             listener.onResponse(new PersistJobAction.Response(false));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
index 88c72578023..dbbaf7b0270 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
@@ -30,7 +30,6 @@ import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -149,7 +148,7 @@ public class TransportPutDatafeedAction extends TransportMasterNodeAction<PutDat
         MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata)
                 .putDatafeed(request.getDatafeed(), headers).build();
         return ClusterState.builder(clusterState).metaData(
-                MetaData.builder(clusterState.getMetaData()).putCustom(MLMetadataField.TYPE, newMetadata).build())
+                MetaData.builder(clusterState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build())
                 .build();
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
index b13ed6d6984..fced6816128 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
@@ -29,8 +29,8 @@ import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedJobValidator;
@@ -89,7 +89,7 @@ public class TransportStartDatafeedAction extends TransportMasterNodeAction<Star
             throw ExceptionsHelper.missingJobException(datafeed.getJobId());
         }
         DatafeedJobValidator.validate(datafeed, job);
-        JobState jobState = MlMetadata.getJobState(datafeed.getJobId(), tasks);
+        JobState jobState = MlTasks.getJobState(datafeed.getJobId(), tasks);
         if (jobState.isAnyOf(JobState.OPENING, JobState.OPENED) == false) {
             throw ExceptionsHelper.conflictStatusException("cannot start datafeed [" + datafeedId + "] because job [" + job.getId() +
                     "] is " + jobState);
@@ -167,7 +167,7 @@ public class TransportStartDatafeedAction extends TransportMasterNodeAction<Star
                                              listener) {
         DataExtractorFactory.create(client, datafeed, job, ActionListener.wrap(
                 dataExtractorFactory ->
-                        persistentTasksService.sendStartRequest(MLMetadataField.datafeedTaskId(params.getDatafeedId()),
+                        persistentTasksService.sendStartRequest(MlTasks.datafeedTaskId(params.getDatafeedId()),
                                 StartDatafeedAction.TASK_NAME, params, listener)
                 , listener::onFailure));
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
index cf7350a870e..1cc6d971589 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
@@ -25,8 +25,8 @@ import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
@@ -79,7 +79,7 @@ public class TransportStopDatafeedAction extends TransportTasksAction<TransportS
         Set<String> expandedDatafeedIds = mlMetadata.expandDatafeedIds(request.getDatafeedId(), request.allowNoDatafeeds());
         for (String expandedDatafeedId : expandedDatafeedIds) {
             validateDatafeedTask(expandedDatafeedId, mlMetadata);
-            addDatafeedTaskIdAccordingToState(expandedDatafeedId, MlMetadata.getDatafeedState(expandedDatafeedId, tasks),
+            addDatafeedTaskIdAccordingToState(expandedDatafeedId, MlTasks.getDatafeedState(expandedDatafeedId, tasks),
                     startedDatafeedIds, stoppingDatafeedIds);
         }
     }
@@ -155,7 +155,7 @@ public class TransportStopDatafeedAction extends TransportTasksAction<TransportS
                                     List<String> startedDatafeeds, List<String> stoppingDatafeeds) {
         Set<String> executorNodes = new HashSet<>();
         for (String datafeedId : startedDatafeeds) {
-            PersistentTasksCustomMetaData.PersistentTask<?> datafeedTask = MlMetadata.getDatafeedTask(datafeedId, tasks);
+            PersistentTasksCustomMetaData.PersistentTask<?> datafeedTask = MlTasks.getDatafeedTask(datafeedId, tasks);
             if (datafeedTask == null || datafeedTask.isAssigned() == false) {
                 String message = "Cannot stop datafeed [" + datafeedId + "] because the datafeed does not have an assigned node." +
                         " Use force stop to stop the datafeed";
@@ -171,8 +171,8 @@ public class TransportStopDatafeedAction extends TransportTasksAction<TransportS
         // wait for started and stopping datafeeds
         // Map datafeedId -> datafeed task Id.
         List<String> allDataFeedsToWaitFor = Stream.concat(
-                startedDatafeeds.stream().map(id -> MLMetadataField.datafeedTaskId(id)),
-                stoppingDatafeeds.stream().map(id -> MLMetadataField.datafeedTaskId(id)))
+                startedDatafeeds.stream().map(MlTasks::datafeedTaskId),
+                stoppingDatafeeds.stream().map(MlTasks::datafeedTaskId))
                 .collect(Collectors.toList());
 
         ActionListener<StopDatafeedAction.Response> finalListener = ActionListener.wrap(
@@ -188,7 +188,7 @@ public class TransportStopDatafeedAction extends TransportTasksAction<TransportS
         final AtomicArray<Exception> failures = new AtomicArray<>(startedDatafeeds.size());
 
         for (String datafeedId : startedDatafeeds) {
-            PersistentTasksCustomMetaData.PersistentTask<?> datafeedTask = MlMetadata.getDatafeedTask(datafeedId, tasks);
+            PersistentTasksCustomMetaData.PersistentTask<?> datafeedTask = MlTasks.getDatafeedTask(datafeedId, tasks);
             if (datafeedTask != null) {
                 persistentTasksService.sendRemoveRequest(datafeedTask.getId(),
                         new ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>>() {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
index 4e43cbb1853..8cf917c4405 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
@@ -19,7 +19,6 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateDatafeedAction;
@@ -76,7 +75,7 @@ public class TransportUpdateDatafeedAction extends TransportMasterNodeAction<Upd
                                 .updateDatafeed(update, persistentTasks, headers).build();
                         updatedDatafeed = newMetadata.getDatafeed(update.getId());
                         return ClusterState.builder(currentState).metaData(
-                                MetaData.builder(currentState.getMetaData()).putCustom(MLMetadataField.TYPE, newMetadata).build()).build();
+                                MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build()).build();
                     }
                 });
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
index 338c111401a..3d4d66eba92 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
@@ -21,6 +21,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -257,7 +258,7 @@ public class DatafeedManager extends AbstractComponent {
     }
 
     private JobState getJobState(PersistentTasksCustomMetaData tasks, TransportStartDatafeedAction.DatafeedTask datafeedTask) {
-        return MlMetadata.getJobState(getJobId(datafeedTask), tasks);
+        return MlTasks.getJobState(getJobId(datafeedTask), tasks);
     }
 
     private TimeValue computeNextDelay(long next) {
@@ -385,7 +386,7 @@ public class DatafeedManager extends AbstractComponent {
         private void closeJob() {
             ClusterState clusterState = clusterService.state();
             PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-            JobState jobState = MlMetadata.getJobState(getJobId(), tasks);
+            JobState jobState = MlTasks.getJobState(getJobId(), tasks);
             if (jobState != JobState.OPENED) {
                 logger.debug("[{}] No need to auto-close job as job state is [{}]", getJobId(), jobState);
                 return;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
index bebf0f3935d..a6be0476486 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
@@ -14,6 +14,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
@@ -35,7 +36,7 @@ public class DatafeedNodeSelector {
         MlMetadata mlMetadata = MlMetadata.getMlMetadata(clusterState);
         PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
         this.datafeed = mlMetadata.getDatafeed(datafeedId);
-        this.jobTask = MlMetadata.getJobTask(datafeed.getJobId(), tasks);
+        this.jobTask = MlTasks.getJobTask(datafeed.getJobId(), tasks);
         this.clusterState = Objects.requireNonNull(clusterState);
         this.resolver = Objects.requireNonNull(resolver);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
index f7fab029c88..fcc9151b755 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
@@ -30,9 +30,9 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.xpack.core.XPackPlugin;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
@@ -169,7 +169,7 @@ public class JobManager extends AbstractComponent {
 
     public JobState getJobState(String jobId) {
         PersistentTasksCustomMetaData tasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        return MlMetadata.getJobState(jobId, tasks);
+        return MlTasks.getJobState(jobId, tasks);
     }
 
     /**
@@ -411,7 +411,7 @@ public class JobManager extends AbstractComponent {
 
     private boolean isJobOpen(ClusterState clusterState, String jobId) {
         PersistentTasksCustomMetaData persistentTasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE);
-        JobState jobState = MlMetadata.getJobState(jobId, persistentTasks);
+        JobState jobState = MlTasks.getJobState(jobId, persistentTasks);
         return jobState == JobState.OPENED;
     }
 
@@ -618,7 +618,7 @@ public class JobManager extends AbstractComponent {
     private static ClusterState buildNewClusterState(ClusterState currentState, MlMetadata.Builder builder) {
         XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
         ClusterState.Builder newState = ClusterState.builder(currentState);
-        newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MLMetadataField.TYPE, builder.build()).build());
+        newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, builder.build()).build());
         return newState.build();
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java
index 5893a863fe3..5278171d438 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java
@@ -24,7 +24,6 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackFeatureSet.Usage;
 import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
@@ -274,7 +273,7 @@ public class MachineLearningFeatureSetTests extends ESTestCase {
         }
         ClusterState clusterState = new ClusterState.Builder(ClusterState.EMPTY_STATE)
                 .metaData(new MetaData.Builder()
-                        .putCustom(MLMetadataField.TYPE, mlMetadataBuilder.build()))
+                        .putCustom(MlMetadata.TYPE, mlMetadataBuilder.build()))
                 .build();
         when(clusterService.state()).thenReturn(clusterState);
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
index d324d17f40e..c7f50440f0e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
@@ -18,7 +18,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.junit.Before;
 
@@ -99,7 +98,7 @@ public class MlInitializationServiceTests extends ESTestCase {
                         .localNodeId("_node_id")
                         .masterNodeId("_node_id"))
                 .metaData(MetaData.builder()
-                        .putCustom(MLMetadataField.TYPE, new MlMetadata.Builder().build()))
+                        .putCustom(MlMetadata.TYPE, new MlMetadata.Builder().build()))
                 .build();
         MlDailyMaintenanceService initialDailyMaintenanceService = mock(MlDailyMaintenanceService.class);
         initializationService.setDailyMaintenanceService(initialDailyMaintenanceService);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
index ecfe7128583..fd35c4b3bad 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
@@ -17,9 +17,8 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
-import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests;
@@ -27,7 +26,6 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
-import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 import org.elasticsearch.xpack.core.ml.job.config.JobTests;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField;
@@ -305,7 +303,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
 
         PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
         StartDatafeedAction.DatafeedParams params = new StartDatafeedAction.DatafeedParams(datafeedConfig1.getId(), 0L);
-        tasksBuilder.addTask(MLMetadataField.datafeedTaskId("datafeed1"), StartDatafeedAction.TASK_NAME, params, INITIAL_ASSIGNMENT);
+        tasksBuilder.addTask(MlTasks.datafeedTaskId("datafeed1"), StartDatafeedAction.TASK_NAME, params, INITIAL_ASSIGNMENT);
         PersistentTasksCustomMetaData tasksInProgress = tasksBuilder.build();
 
         DatafeedUpdate.Builder update = new DatafeedUpdate.Builder(datafeedConfig1.getId());
@@ -387,7 +385,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
 
         PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
         StartDatafeedAction.DatafeedParams params = new StartDatafeedAction.DatafeedParams("datafeed1", 0L);
-        tasksBuilder.addTask(MLMetadataField.datafeedTaskId("datafeed1"), StartDatafeedAction.TASK_NAME, params, INITIAL_ASSIGNMENT);
+        tasksBuilder.addTask(MlTasks.datafeedTaskId("datafeed1"), StartDatafeedAction.TASK_NAME, params, INITIAL_ASSIGNMENT);
         PersistentTasksCustomMetaData tasksInProgress = tasksBuilder.build();
 
         MlMetadata.Builder builder2 = new MlMetadata.Builder(result);
@@ -396,19 +394,6 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
         assertThat(e.status(), equalTo(RestStatus.CONFLICT));
     }
 
-    public void testGetJobState() {
-        PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
-        // A missing task is a closed job
-        assertEquals(JobState.CLOSED, MlMetadata.getJobState("foo", tasksBuilder.build()));
-        // A task with no status is opening
-        tasksBuilder.addTask(MlMetadata.jobTaskId("foo"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("foo"),
-                new PersistentTasksCustomMetaData.Assignment("bar", "test assignment"));
-        assertEquals(JobState.OPENING, MlMetadata.getJobState("foo", tasksBuilder.build()));
-
-        tasksBuilder.updateTaskState(MlMetadata.jobTaskId("foo"), new JobTaskState(JobState.OPENED, tasksBuilder.getLastAllocationId()));
-        assertEquals(JobState.OPENED, MlMetadata.getJobState("foo", tasksBuilder.build()));
-    }
-
     public void testExpandJobIds() {
         MlMetadata mlMetadata = newMlMetadataWithJobs("bar-1", "foo-1", "foo-2").build();
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java
new file mode 100644
index 00000000000..687292b3c85
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlTasksTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ml;
+
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
+import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
+import org.elasticsearch.xpack.core.ml.job.config.JobState;
+import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
+
+public class MlTasksTests extends ESTestCase {
+    public void testGetJobState() {
+        PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
+        // A missing task is a closed job
+        assertEquals(JobState.CLOSED, MlTasks.getJobState("foo", tasksBuilder.build()));
+        // A task with no status is opening
+        tasksBuilder.addTask(MlTasks.jobTaskId("foo"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("foo"),
+                new PersistentTasksCustomMetaData.Assignment("bar", "test assignment"));
+        assertEquals(JobState.OPENING, MlTasks.getJobState("foo", tasksBuilder.build()));
+
+        tasksBuilder.updateTaskState(MlTasks.jobTaskId("foo"), new JobTaskState(JobState.OPENED, tasksBuilder.getLastAllocationId()));
+        assertEquals(JobState.OPENED, MlTasks.getJobState("foo", tasksBuilder.build()));
+    }
+
+    public void testGetDatefeedState() {
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        // A missing task is a stopped datafeed
+        assertEquals(DatafeedState.STOPPED, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
+
+        tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), StartDatafeedAction.TASK_NAME,
+                new StartDatafeedAction.DatafeedParams("foo", 0L),
+                new PersistentTasksCustomMetaData.Assignment("bar", "test assignment"));
+        assertEquals(DatafeedState.STOPPED, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
+
+        tasksBuilder.updateTaskState(MlTasks.datafeedTaskId("foo"), DatafeedState.STARTED);
+        assertEquals(DatafeedState.STARTED, MlTasks.getDatafeedState("foo", tasksBuilder.build()));
+    }
+
+    public void testGetJobTask() {
+        assertNull(MlTasks.getJobTask("foo", null));
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
+        tasksBuilder.addTask(MlTasks.jobTaskId("foo"), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams("foo"),
+                new PersistentTasksCustomMetaData.Assignment("bar", "test assignment"));
+
+        assertNotNull(MlTasks.getJobTask("foo", tasksBuilder.build()));
+        assertNull(MlTasks.getJobTask("other", tasksBuilder.build()));
+    }
+
+    public void testGetDatafeedTask() {
+        assertNull(MlTasks.getDatafeedTask("foo", null));
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), StartDatafeedAction.TASK_NAME,
+                new StartDatafeedAction.DatafeedParams("foo", 0L),
+                new PersistentTasksCustomMetaData.Assignment("bar", "test assignment"));
+
+        assertNotNull(MlTasks.getDatafeedTask("foo", tasksBuilder.build()));
+        assertNull(MlTasks.getDatafeedTask("other", tasksBuilder.build()));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java
index 27d3a68d005..98b84ed8113 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java
@@ -20,8 +20,8 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction.Request;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
@@ -107,7 +107,7 @@ public class TransportCloseJobActionTests extends ESTestCase {
         addJobTask("job_id_4", null, JobState.CLOSING, tasksBuilder);
 
         ClusterState cs1 = ClusterState.builder(new ClusterName("_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlBuilder.build())
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
                         .putCustom(PersistentTasksCustomMetaData.TYPE,  tasksBuilder.build()))
                 .build();
 
@@ -133,7 +133,7 @@ public class TransportCloseJobActionTests extends ESTestCase {
         addJobTask("job_id_1", null, JobState.OPENED, tasksBuilder);
 
         ClusterState cs1 = ClusterState.builder(new ClusterName("_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlBuilder.build())
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
                         .putCustom(PersistentTasksCustomMetaData.TYPE,  tasksBuilder.build()))
                 .build();
 
@@ -147,7 +147,7 @@ public class TransportCloseJobActionTests extends ESTestCase {
 
         // Job without task is closed
         cs1 = ClusterState.builder(new ClusterName("_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlBuilder.build()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()))
                 .build();
 
         openJobs.clear();
@@ -162,7 +162,7 @@ public class TransportCloseJobActionTests extends ESTestCase {
         mlBuilder.putJob(BaseMlIntegTestCase.createFareQuoteJob("job_id_1").build(new Date()), false);
 
         ClusterState cs1 = ClusterState.builder(new ClusterName("_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlBuilder.build()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build()))
                 .build();
 
         List<String> openJobs = new ArrayList<>();
@@ -181,7 +181,7 @@ public class TransportCloseJobActionTests extends ESTestCase {
         addJobTask("job_id_failed", null, JobState.FAILED, tasksBuilder);
 
         ClusterState cs1 = ClusterState.builder(new ClusterName("_name")).metaData(new MetaData.Builder()
-                .putCustom(MLMetadataField.TYPE, mlBuilder.build()).putCustom(PersistentTasksCustomMetaData.TYPE,
+                .putCustom(MlMetadata.TYPE, mlBuilder.build()).putCustom(PersistentTasksCustomMetaData.TYPE,
                         tasksBuilder.build())).build();
 
         List<String> openJobs = new ArrayList<>();
@@ -216,7 +216,7 @@ public class TransportCloseJobActionTests extends ESTestCase {
         // closed job has no task
 
         ClusterState cs1 = ClusterState.builder(new ClusterName("_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlBuilder.build())
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
                         .putCustom(PersistentTasksCustomMetaData.TYPE,  tasksBuilder.build()))
                 .build();
 
@@ -256,7 +256,7 @@ public class TransportCloseJobActionTests extends ESTestCase {
         addJobTask("foo", null, JobState.CLOSED, tasksBuilder);
 
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlBuilder.build())
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlBuilder.build())
                         .putCustom(PersistentTasksCustomMetaData.TYPE,  tasksBuilder.build()))
                 .build();
 
@@ -311,9 +311,9 @@ public class TransportCloseJobActionTests extends ESTestCase {
 
     public static void addTask(String datafeedId, long startTime, String nodeId, DatafeedState state,
                                PersistentTasksCustomMetaData.Builder tasks) {
-        tasks.addTask(MLMetadataField.datafeedTaskId(datafeedId), StartDatafeedAction.TASK_NAME,
+        tasks.addTask(MlTasks.datafeedTaskId(datafeedId), StartDatafeedAction.TASK_NAME,
                 new StartDatafeedAction.DatafeedParams(datafeedId, startTime), new Assignment(nodeId, "test assignment"));
-        tasks.updateTaskState(MLMetadataField.datafeedTaskId(datafeedId), state);
+        tasks.updateTaskState(MlTasks.datafeedTaskId(datafeedId), state);
     }
 
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobActionTests.java
index 1bf40ac881e..7464348adb9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobActionTests.java
@@ -9,7 +9,6 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
 
@@ -21,7 +20,7 @@ public class TransportDeleteJobActionTests extends ESTestCase {
         MlMetadata mlMetadata = MlMetadata.EMPTY_METADATA;
 
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlMetadata))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata))
                 .build();
 
         assertTrue(TransportDeleteJobAction.jobIsDeletedFromState("job_id_1", clusterState));
@@ -30,7 +29,7 @@ public class TransportDeleteJobActionTests extends ESTestCase {
         mlBuilder.putJob(BaseMlIntegTestCase.createScheduledJob("job_id_1").build(new Date()), false);
         mlMetadata = mlBuilder.build();
         clusterState = ClusterState.builder(new ClusterName("_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlMetadata))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata))
                 .build();
 
         assertFalse(TransportDeleteJobAction.jobIsDeletedFromState("job_id_1", clusterState));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java
index dd8ddf3aa62..02bfb1b326f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java
@@ -32,9 +32,9 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -317,7 +317,7 @@ public class TransportOpenJobActionTests extends ESTestCase {
         assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
 
         tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
-        tasksBuilder.reassignTask(MlMetadata.jobTaskId("job_id6"), new Assignment("_node_id3", "test assignment"));
+        tasksBuilder.reassignTask(MlTasks.jobTaskId("job_id6"), new Assignment("_node_id3", "test assignment"));
         tasks = tasksBuilder.build();
 
         csBuilder = ClusterState.builder(cs);
@@ -328,7 +328,7 @@ public class TransportOpenJobActionTests extends ESTestCase {
         assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
 
         tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
-        tasksBuilder.updateTaskState(MlMetadata.jobTaskId("job_id6"), null);
+        tasksBuilder.updateTaskState(MlTasks.jobTaskId("job_id6"), null);
         tasks = tasksBuilder.build();
 
         csBuilder = ClusterState.builder(cs);
@@ -354,7 +354,7 @@ public class TransportOpenJobActionTests extends ESTestCase {
         PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
         addJobTask("job_id1", "_node_id1", JobState.fromString("failed"), tasksBuilder);
         // This will make the allocation stale for job_id1
-        tasksBuilder.reassignTask(MlMetadata.jobTaskId("job_id1"), new Assignment("_node_id1", "test assignment"));
+        tasksBuilder.reassignTask(MlTasks.jobTaskId("job_id1"), new Assignment("_node_id1", "test assignment"));
         addJobTask("job_id2", "_node_id1", null, tasksBuilder);
         addJobTask("job_id3", "_node_id2", null, tasksBuilder);
         addJobTask("job_id4", "_node_id2", null, tasksBuilder);
@@ -675,10 +675,10 @@ public class TransportOpenJobActionTests extends ESTestCase {
     }
 
     public static void addJobTask(String jobId, String nodeId, JobState jobState, PersistentTasksCustomMetaData.Builder builder) {
-        builder.addTask(MlMetadata.jobTaskId(jobId), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams(jobId),
+        builder.addTask(MlTasks.jobTaskId(jobId), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams(jobId),
                 new Assignment(nodeId, "test assignment"));
         if (jobState != null) {
-            builder.updateTaskState(MlMetadata.jobTaskId(jobId), new JobTaskState(jobState, builder.getLastAllocationId()));
+            builder.updateTaskState(MlTasks.jobTaskId(jobId), new JobTaskState(jobState, builder.getLastAllocationId()));
         }
     }
 
@@ -717,7 +717,7 @@ public class TransportOpenJobActionTests extends ESTestCase {
             Job job = jobCreator.apply(jobId);
             mlMetadata.putJob(job, false);
         }
-        metaData.putCustom(MLMetadataField.TYPE, mlMetadata.build());
+        metaData.putCustom(MlMetadata.TYPE, mlMetadata.build());
     }
 
     private ClusterState getClusterStateWithMappingsWithMetaData(Map<String, Object> namesAndVersions) throws IOException {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java
index 934642986de..a15c0e97b97 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedActionTests.java
@@ -7,8 +7,8 @@ package org.elasticsearch.xpack.ml.action;
 
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -29,9 +29,9 @@ import static org.hamcrest.Matchers.equalTo;
 public class TransportStopDatafeedActionTests extends ESTestCase {
     public void testValidate() {
         PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        tasksBuilder.addTask(MLMetadataField.datafeedTaskId("foo"), StartDatafeedAction.TASK_NAME,
+        tasksBuilder.addTask(MlTasks.datafeedTaskId("foo"), StartDatafeedAction.TASK_NAME,
                 new StartDatafeedAction.DatafeedParams("foo", 0L), new PersistentTasksCustomMetaData.Assignment("node_id", ""));
-        tasksBuilder.updateTaskState(MLMetadataField.datafeedTaskId("foo"), DatafeedState.STARTED);
+        tasksBuilder.updateTaskState(MlTasks.datafeedTaskId("foo"), DatafeedState.STARTED);
         tasksBuilder.build();
 
         Job job = createDatafeedJob().build(new Date());
@@ -118,9 +118,9 @@ public class TransportStopDatafeedActionTests extends ESTestCase {
 
     public static void addTask(String datafeedId, long startTime, String nodeId, DatafeedState state,
                                PersistentTasksCustomMetaData.Builder taskBuilder) {
-        taskBuilder.addTask(MLMetadataField.datafeedTaskId(datafeedId), StartDatafeedAction.TASK_NAME,
+        taskBuilder.addTask(MlTasks.datafeedTaskId(datafeedId), StartDatafeedAction.TASK_NAME,
                 new StartDatafeedAction.DatafeedParams(datafeedId, startTime),
                 new PersistentTasksCustomMetaData.Assignment(nodeId, "test assignment"));
-        taskBuilder.updateTaskState(MLMetadataField.datafeedTaskId(datafeedId), state);
+        taskBuilder.updateTaskState(MlTasks.datafeedTaskId(datafeedId), state);
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java
index 6ce03d22b64..a9dec7c66d4 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java
@@ -23,7 +23,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
@@ -93,7 +92,7 @@ public class DatafeedManagerTests extends ESTestCase {
                         Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
                 .build();
         ClusterState.Builder cs = ClusterState.builder(new ClusterName("cluster_name"))
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, mlMetadata.build())
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, mlMetadata.build())
                         .putCustom(PersistentTasksCustomMetaData.TYPE, tasks))
                 .nodes(nodes);
 
@@ -254,7 +253,7 @@ public class DatafeedManagerTests extends ESTestCase {
         PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
         addJobTask("job_id", "node_id", JobState.OPENING, tasksBuilder);
         ClusterState.Builder cs = ClusterState.builder(clusterService.state())
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
                     .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()));
         when(clusterService.state()).thenReturn(cs.build());
 
@@ -269,7 +268,7 @@ public class DatafeedManagerTests extends ESTestCase {
         addJobTask("job_id", "node_id", JobState.OPENING, tasksBuilder);
         addJobTask("another_job", "node_id", JobState.OPENED, tasksBuilder);
         ClusterState.Builder anotherJobCs = ClusterState.builder(clusterService.state())
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
                     .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()));
 
         capturedClusterStateListener.getValue().clusterChanged(new ClusterChangedEvent("_source", anotherJobCs.build(), cs.build()));
@@ -280,7 +279,7 @@ public class DatafeedManagerTests extends ESTestCase {
         tasksBuilder =  PersistentTasksCustomMetaData.builder();
         addJobTask("job_id", "node_id", JobState.OPENED, tasksBuilder);
         ClusterState.Builder jobOpenedCs = ClusterState.builder(clusterService.state())
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
                     .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()));
 
         capturedClusterStateListener.getValue().clusterChanged(
@@ -294,7 +293,7 @@ public class DatafeedManagerTests extends ESTestCase {
         PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
         addJobTask("job_id", "node_id", JobState.OPENING, tasksBuilder);
         ClusterState.Builder cs = ClusterState.builder(clusterService.state())
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
                     .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()));
         when(clusterService.state()).thenReturn(cs.build());
 
@@ -308,7 +307,7 @@ public class DatafeedManagerTests extends ESTestCase {
         tasksBuilder =  PersistentTasksCustomMetaData.builder();
         addJobTask("job_id", "node_id", JobState.FAILED, tasksBuilder);
         ClusterState.Builder updatedCs = ClusterState.builder(clusterService.state())
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
                     .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()));
 
         capturedClusterStateListener.getValue().clusterChanged(new ClusterChangedEvent("_source", updatedCs.build(), cs.build()));
@@ -322,7 +321,7 @@ public class DatafeedManagerTests extends ESTestCase {
         PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
         addJobTask("job_id", "node_id", JobState.OPENING, tasksBuilder);
         ClusterState.Builder cs = ClusterState.builder(clusterService.state())
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
                     .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()));
         when(clusterService.state()).thenReturn(cs.build());
 
@@ -340,7 +339,7 @@ public class DatafeedManagerTests extends ESTestCase {
         tasksBuilder =  PersistentTasksCustomMetaData.builder();
         addJobTask("job_id", "node_id", JobState.OPENED, tasksBuilder);
         ClusterState.Builder updatedCs = ClusterState.builder(clusterService.state())
-                .metaData(new MetaData.Builder().putCustom(MLMetadataField.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
+                .metaData(new MetaData.Builder().putCustom(MlMetadata.TYPE, MlMetadata.getMlMetadata(clusterService.state()))
                     .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()));
 
         capturedClusterStateListener.getValue().clusterChanged(new ClusterChangedEvent("_source", cs.build(), updatedCs.build()));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
index f3fa804bb27..3a6082c6cf0 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
@@ -27,8 +27,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
@@ -257,7 +257,7 @@ public class DatafeedNodeSelectorTests extends ESTestCase {
         PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
         addJobTask(job.getId(), nodeId, JobState.OPENED, tasksBuilder);
         // Set to lower allocationId, so job task is stale:
-        tasksBuilder.updateTaskState(MlMetadata.jobTaskId(job.getId()), new JobTaskState(JobState.OPENED, 0));
+        tasksBuilder.updateTaskState(MlTasks.jobTaskId(job.getId()), new JobTaskState(JobState.OPENED, 0));
         tasks = tasksBuilder.build();
 
         givenClusterState("foo", 1, 0);
@@ -319,7 +319,7 @@ public class DatafeedNodeSelectorTests extends ESTestCase {
 
         clusterState = ClusterState.builder(new ClusterName("cluster_name"))
                 .metaData(new MetaData.Builder()
-                        .putCustom(MLMetadataField.TYPE, mlMetadata)
+                        .putCustom(MlMetadata.TYPE, mlMetadata)
                         .putCustom(PersistentTasksCustomMetaData.TYPE, tasks)
                         .put(indexMetaData, false))
                 .nodes(nodes)
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
index 83facf4c1f2..9ab4907b2cd 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
@@ -23,7 +23,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
@@ -206,7 +206,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
         assertBusy(() -> {
             ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
             PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-            PersistentTask<?> task = tasks.getTask(MlMetadata.jobTaskId(jobId));
+            PersistentTask<?> task = tasks.getTask(MlTasks.jobTaskId(jobId));
 
             DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode());
             assertThat(node.getAttributes(), hasEntry(MachineLearning.ML_ENABLED_NODE_ATTR, "true"));
@@ -390,7 +390,7 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase {
         ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
         PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
         assertEquals(1, tasks.taskMap().size());
-        PersistentTask<?> task = MlMetadata.getJobTask(jobId, tasks);
+        PersistentTask<?> task = MlTasks.getJobTask(jobId, tasks);
         assertNotNull(task);
 
         if (hasExecutorNode) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java
index 14ec4813a74..f389bc4b749 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java
@@ -9,7 +9,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
@@ -110,7 +109,7 @@ public class DeleteJobIT extends BaseMlIntegTestCase {
         builder.markJobAsDeleted(jobId, tasks, true);
 
         ClusterState.Builder newState = ClusterState.builder(currentState);
-        return newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MLMetadataField.TYPE, builder.build()).build())
+        return newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, builder.build()).build())
                 .build();
     }
 
@@ -119,7 +118,7 @@ public class DeleteJobIT extends BaseMlIntegTestCase {
         builder.deleteJob(jobId, currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE));
 
         ClusterState.Builder newState = ClusterState.builder(currentState);
-        return newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MLMetadataField.TYPE, builder.build()).build())
+        return newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, builder.build()).build())
                 .build();
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
index 68ce5da5ca1..1b1c39c3bcf 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java
@@ -13,7 +13,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
@@ -57,7 +57,7 @@ public class TooManyJobsIT extends BaseMlIntegTestCase {
         PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
         assertEquals(1, tasks.taskMap().size());
         // now just double check that the first job is still opened:
-        PersistentTasksCustomMetaData.PersistentTask task = tasks.getTask(MlMetadata.jobTaskId("close-failed-job-1"));
+        PersistentTasksCustomMetaData.PersistentTask task = tasks.getTask(MlTasks.jobTaskId("close-failed-job-1"));
         assertEquals(JobState.OPENED, ((JobTaskState) task.getState()).getState());
     }
 
@@ -121,7 +121,7 @@ public class TooManyJobsIT extends BaseMlIntegTestCase {
                     for (Client client : clients()) {
                         PersistentTasksCustomMetaData tasks = client.admin().cluster().prepareState().get().getState()
                                 .getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-                        assertEquals(MlMetadata.getJobState(job.getId(), tasks), JobState.OPENED);
+                        assertEquals(MlTasks.getJobState(job.getId(), tasks), JobState.OPENED);
                     }
                 });
                 return;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
index cf925963c19..e15b5828df9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
@@ -21,7 +21,6 @@ import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
@@ -94,7 +93,7 @@ public class JobManagerTests extends ESTestCase {
         Job job = buildJobBuilder("foo").build();
         MlMetadata mlMetadata = new MlMetadata.Builder().putJob(job, false).build();
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
-                .metaData(MetaData.builder().putCustom(MLMetadataField.TYPE, mlMetadata)).build();
+                .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata)).build();
 
         assertEquals(job, JobManager.getJobOrThrowIfUnknown("foo", cs));
     }
@@ -105,7 +104,7 @@ public class JobManagerTests extends ESTestCase {
             mlMetadata.putJob(buildJobBuilder(Integer.toString(i)).build(), false);
         }
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
-                .metaData(MetaData.builder().putCustom(MLMetadataField.TYPE, mlMetadata.build())).build();
+                .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata.build())).build();
 
         JobManager jobManager = createJobManager();
         QueryPage<Job> result = jobManager.expandJobs("_all", true, clusterState);
@@ -161,7 +160,7 @@ public class JobManagerTests extends ESTestCase {
         MlMetadata.Builder mlMetadata = new MlMetadata.Builder();
         mlMetadata.putJob(buildJobBuilder("foo").build(), false);
         ClusterState clusterState = ClusterState.builder(new ClusterName("name"))
-                .metaData(MetaData.builder().putCustom(MLMetadataField.TYPE, mlMetadata.build())).build();
+                .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata.build())).build();
 
         jobManager.putJob(putJobRequest, analysisRegistry, clusterState, new ActionListener<PutJobAction.Response>() {
             @Override
@@ -215,7 +214,7 @@ public class JobManagerTests extends ESTestCase {
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
                 .metaData(MetaData.builder()
                         .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())
-                        .putCustom(MLMetadataField.TYPE, mlMetadata.build()))
+                        .putCustom(MlMetadata.TYPE, mlMetadata.build()))
                 .build();
         when(clusterService.state()).thenReturn(clusterState);
 
@@ -267,7 +266,7 @@ public class JobManagerTests extends ESTestCase {
 
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
                 .metaData(MetaData.builder()
-                        .putCustom(MLMetadataField.TYPE, mlMetadata.build()))
+                        .putCustom(MlMetadata.TYPE, mlMetadata.build()))
                 .build();
         when(clusterService.state()).thenReturn(clusterState);
 
@@ -297,7 +296,7 @@ public class JobManagerTests extends ESTestCase {
 
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
                 .metaData(MetaData.builder()
-                        .putCustom(MLMetadataField.TYPE, mlMetadata.build()))
+                        .putCustom(MlMetadata.TYPE, mlMetadata.build()))
                 .build();
         when(clusterService.state()).thenReturn(clusterState);
 
@@ -331,7 +330,7 @@ public class JobManagerTests extends ESTestCase {
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
                 .metaData(MetaData.builder()
                         .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())
-                        .putCustom(MLMetadataField.TYPE, mlMetadata.build()))
+                        .putCustom(MlMetadata.TYPE, mlMetadata.build()))
                 .build();
         when(clusterService.state()).thenReturn(clusterState);
 
@@ -370,7 +369,7 @@ public class JobManagerTests extends ESTestCase {
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
                 .metaData(MetaData.builder()
                         .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())
-                        .putCustom(MLMetadataField.TYPE, mlMetadata.build()))
+                        .putCustom(MlMetadata.TYPE, mlMetadata.build()))
                 .build();
         when(clusterService.state()).thenReturn(clusterState);
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemoverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemoverTests.java
index 82441c42467..4195e93439d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemoverTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemoverTests.java
@@ -24,7 +24,6 @@ import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
@@ -212,7 +211,7 @@ public class ExpiredModelSnapshotsRemoverTests extends ESTestCase {
         MlMetadata mlMetadata = mock(MlMetadata.class);
         when(mlMetadata.getJobs()).thenReturn(jobsMap);
         MetaData metadata = mock(MetaData.class);
-        when(metadata.custom(MLMetadataField.TYPE)).thenReturn(mlMetadata);
+        when(metadata.custom(MlMetadata.TYPE)).thenReturn(mlMetadata);
         when(clusterState.getMetaData()).thenReturn(metadata);
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemoverTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemoverTests.java
index 2f08574e1a6..af9ec8b84a6 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemoverTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemoverTests.java
@@ -18,7 +18,6 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.mock.orig.Mockito;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobTests;
@@ -165,7 +164,7 @@ public class ExpiredResultsRemoverTests extends ESTestCase {
         MlMetadata mlMetadata = mock(MlMetadata.class);
         when(mlMetadata.getJobs()).thenReturn(jobsMap);
         MetaData metadata = mock(MetaData.class);
-        when(metadata.custom(MLMetadataField.TYPE)).thenReturn(mlMetadata);
+        when(metadata.custom(MlMetadata.TYPE)).thenReturn(mlMetadata);
         when(clusterState.getMetaData()).thenReturn(metadata);
     }
 

From a525c36c6092de16a0260ffe3c5b5b2e705f65c4 Mon Sep 17 00:00:00 2001
From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com>
Date: Wed, 25 Jul 2018 00:44:26 +1000
Subject: [PATCH 156/260] [Kerberos] Add Kerberos authentication support
 (#32263)

This commit adds support for Kerberos authentication with a platinum
license. Kerberos authentication support relies on SPNEGO, which is
triggered by challenging clients with a 401 response with the
`WWW-Authenticate: Negotiate` header. A SPNEGO client will then provide
a Kerberos ticket in the `Authorization` header. The tickets are
validated using Java's built-in GSS support. The JVM uses a vm wide
configuration for Kerberos, so there can be only one Kerberos realm.
This is enforced by a bootstrap check that also enforces the existence
of the keytab file.

In many cases a fallback authentication mechanism is needed when SPNEGO
authentication is not available. In order to support this, the
DefaultAuthenticationFailureHandler now takes a list of failure response
headers. For example, one realm can provide a
`WWW-Authenticate: Negotiate` header as its default and another could
provide `WWW-Authenticate: Basic` to indicate to the client that basic
authentication can be used in place of SPNEGO.

In order to test Kerberos, unit tests are run against an in-memory KDC
that is backed by an in-memory ldap server. A QA project has also been
added to test against an actual KDC, which is provided by the krb5kdc
fixture.

Closes #30243
---
 .../src/main/resources/provision/addprinc.sh  |  22 +-
 .../DefaultAuthenticationFailureHandler.java  | 118 +++++--
 .../xpack/core/security/authc/Realm.java      |  15 +
 .../authc/kerberos/KerberosRealmSettings.java |  50 +++
 ...aultAuthenticationFailureHandlerTests.java | 114 +++++++
 x-pack/plugin/security/build.gradle           |  63 +++-
 .../xpack/security/Security.java              |  62 +++-
 .../security/authc/AuthenticationService.java |   4 +-
 .../xpack/security/authc/InternalRealms.java  |  16 +-
 .../kerberos/KerberosAuthenticationToken.java | 150 +++++++++
 .../authc/kerberos/KerberosRealm.java         | 214 ++++++++++++
 .../kerberos/KerberosRealmBootstrapCheck.java |  69 ++++
 .../kerberos/KerberosTicketValidator.java     | 273 +++++++++++++++
 .../plugin-metadata/plugin-security.policy    |  15 +
 .../authc/AuthenticationServiceTests.java     |  56 ++++
 .../security/authc/InternalRealmsTests.java   |  13 +
 .../xpack/security/authc/RealmsTests.java     |   8 +-
 .../KerberosAuthenticationTokenTests.java     | 110 ++++++
 .../KerberosRealmAuthenticateFailedTests.java | 108 ++++++
 .../KerberosRealmBootstrapCheckTests.java     | 114 +++++++
 .../kerberos/KerberosRealmCacheTests.java     | 141 ++++++++
 .../kerberos/KerberosRealmSettingsTests.java  |  46 +++
 .../authc/kerberos/KerberosRealmTestCase.java | 168 ++++++++++
 .../authc/kerberos/KerberosRealmTests.java    |  97 ++++++
 .../authc/kerberos/KerberosTestCase.java      | 223 ++++++++++++
 .../KerberosTicketValidatorTests.java         | 133 ++++++++
 .../authc/kerberos/SimpleKdcLdapServer.java   | 224 +++++++++++++
 .../kerberos/SimpleKdcLdapServerTests.java    |  77 +++++
 .../security/authc/kerberos/SpnegoClient.java | 257 ++++++++++++++
 .../security/src/test/resources/kdc.ldiff     |  23 ++
 x-pack/qa/kerberos-tests/build.gradle         | 127 +++++++
 .../kerberos/KerberosAuthenticationIT.java    | 152 +++++++++
 ...SpnegoHttpClientConfigCallbackHandler.java | 317 ++++++++++++++++++
 .../src/test/resources/plugin-security.policy |   4 +
 34 files changed, 3522 insertions(+), 61 deletions(-)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/kerberos/KerberosRealmSettings.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationToken.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidator.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationTokenTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmSettingsTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidatorTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServerTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
 create mode 100644 x-pack/plugin/security/src/test/resources/kdc.ldiff
 create mode 100644 x-pack/qa/kerberos-tests/build.gradle
 create mode 100644 x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
 create mode 100644 x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
 create mode 100644 x-pack/qa/kerberos-tests/src/test/resources/plugin-security.policy

diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh
index 137135dc2aa..d0d1570ae29 100755
--- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh
@@ -20,11 +20,14 @@
 set -e
 
 if [[ $# -lt 1 ]]; then
-  echo 'Usage: addprinc.sh <principalNameNoRealm>'
+  echo 'Usage: addprinc.sh principalName [password]'
+  echo '  principalName    user principal name without realm'
+  echo '  password         If provided then will set password for user else it will provision user with keytab'
   exit 1
 fi
 
 PRINC="$1"
+PASSWD="$2"
 USER=$(echo $PRINC | tr "/" "_")
 
 VDIR=/vagrant
@@ -47,12 +50,17 @@ ADMIN_KTAB=$LOCALSTATEDIR/admin.keytab
 USER_PRIN=$PRINC@$REALM
 USER_KTAB=$LOCALSTATEDIR/$USER.keytab
 
-if [ -f $USER_KTAB ]; then
+if [ -f $USER_KTAB ] && [ -z "$PASSWD" ]; then
   echo "Principal '${PRINC}@${REALM}' already exists. Re-copying keytab..."
+  sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
 else
-  echo "Provisioning '${PRINC}@${REALM}' principal and keytab..."
-  sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
-  sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
+  if [ -z "$PASSWD" ]; then
+    echo "Provisioning '${PRINC}@${REALM}' principal and keytab..."
+    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
+    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
+    sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
+  else
+    echo "Provisioning '${PRINC}@${REALM}' principal with password..."
+    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC"
+  fi
 fi
-
-sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java
index 8b31e77f9f8..d6f678a2dcb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java
@@ -10,60 +10,132 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TransportMessage;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.xpack.core.security.support.Exceptions.authenticationError;
 
 /**
- * The default implementation of a {@link AuthenticationFailureHandler}. This handler will return an exception with a
- * RestStatus of 401 and the WWW-Authenticate header with a Basic challenge.
+ * The default implementation of a {@link AuthenticationFailureHandler}. This
+ * handler will return an exception with a RestStatus of 401 and default failure
+ * response headers like 'WWW-Authenticate'
  */
 public class DefaultAuthenticationFailureHandler implements AuthenticationFailureHandler {
+    private final Map<String, List<String>> defaultFailureResponseHeaders;
+
+    /**
+     * Constructs default authentication failure handler
+     *
+     * @deprecated replaced by {@link #DefaultAuthenticationFailureHandler(Map)}
+     */
+    @Deprecated
+    public DefaultAuthenticationFailureHandler() {
+        this(null);
+    }
+
+    /**
+     * Constructs default authentication failure handler with provided default
+     * response headers.
+     *
+     * @param failureResponseHeaders Map of header key and list of header values to
+     *            be sent as failure response.
+     * @see Realm#getAuthenticationFailureHeaders()
+     */
+    public DefaultAuthenticationFailureHandler(Map<String, List<String>> failureResponseHeaders) {
+        if (failureResponseHeaders == null || failureResponseHeaders.isEmpty()) {
+            failureResponseHeaders = Collections.singletonMap("WWW-Authenticate",
+                    Collections.singletonList("Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""));
+        }
+        this.defaultFailureResponseHeaders = Collections.unmodifiableMap(failureResponseHeaders);
+    }
 
     @Override
-    public ElasticsearchSecurityException failedAuthentication(RestRequest request, AuthenticationToken token,
-                                                               ThreadContext context) {
-        return authenticationError("unable to authenticate user [{}] for REST request [{}]", token.principal(), request.uri());
+    public ElasticsearchSecurityException failedAuthentication(RestRequest request, AuthenticationToken token, ThreadContext context) {
+        return createAuthenticationError("unable to authenticate user [{}] for REST request [{}]", null, token.principal(), request.uri());
     }
 
     @Override
     public ElasticsearchSecurityException failedAuthentication(TransportMessage message, AuthenticationToken token, String action,
-                                                               ThreadContext context) {
-        return authenticationError("unable to authenticate user [{}] for action [{}]", token.principal(), action);
+            ThreadContext context) {
+        return createAuthenticationError("unable to authenticate user [{}] for action [{}]", null, token.principal(), action);
     }
 
     @Override
     public ElasticsearchSecurityException exceptionProcessingRequest(RestRequest request, Exception e, ThreadContext context) {
-        if (e instanceof ElasticsearchSecurityException) {
-            assert ((ElasticsearchSecurityException) e).status() == RestStatus.UNAUTHORIZED;
-            assert ((ElasticsearchSecurityException) e).getHeader("WWW-Authenticate").size() == 1;
-            return (ElasticsearchSecurityException) e;
-        }
-        return authenticationError("error attempting to authenticate request", e);
+        return createAuthenticationError("error attempting to authenticate request", e, (Object[]) null);
     }
 
     @Override
     public ElasticsearchSecurityException exceptionProcessingRequest(TransportMessage message, String action, Exception e,
-                                                                     ThreadContext context) {
-        if (e instanceof ElasticsearchSecurityException) {
-            assert ((ElasticsearchSecurityException) e).status() == RestStatus.UNAUTHORIZED;
-            assert ((ElasticsearchSecurityException) e).getHeader("WWW-Authenticate").size() == 1;
-            return (ElasticsearchSecurityException) e;
-        }
-        return authenticationError("error attempting to authenticate request", e);
+            ThreadContext context) {
+        return createAuthenticationError("error attempting to authenticate request", e, (Object[]) null);
     }
 
     @Override
     public ElasticsearchSecurityException missingToken(RestRequest request, ThreadContext context) {
-        return authenticationError("missing authentication token for REST request [{}]", request.uri());
+        return createAuthenticationError("missing authentication token for REST request [{}]", null, request.uri());
     }
 
     @Override
     public ElasticsearchSecurityException missingToken(TransportMessage message, String action, ThreadContext context) {
-        return authenticationError("missing authentication token for action [{}]", action);
+        return createAuthenticationError("missing authentication token for action [{}]", null, action);
     }
 
     @Override
     public ElasticsearchSecurityException authenticationRequired(String action, ThreadContext context) {
-        return authenticationError("action [{}] requires authentication", action);
+        return createAuthenticationError("action [{}] requires authentication", null, action);
+    }
+
+    /**
+     * Creates an instance of {@link ElasticsearchSecurityException} with
+     * {@link RestStatus#UNAUTHORIZED} status.
+     * <p>
+     * Also adds default failure response headers as configured for this
+     * {@link DefaultAuthenticationFailureHandler}
+     * <p>
+     * It may replace existing response headers if the cause is an instance of
+     * {@link ElasticsearchSecurityException}
+     *
+     * @param message error message
+     * @param t cause, if it is an instance of
+     *            {@link ElasticsearchSecurityException} asserts status is
+     *            RestStatus.UNAUTHORIZED and adds headers to it, else it will
+     *            create a new instance of {@link ElasticsearchSecurityException}
+     * @param args error message args
+     * @return instance of {@link ElasticsearchSecurityException}
+     */
+    private ElasticsearchSecurityException createAuthenticationError(final String message, final Throwable t, final Object... args) {
+        final ElasticsearchSecurityException ese;
+        final boolean containsNegotiateWithToken;
+        if (t instanceof ElasticsearchSecurityException) {
+            assert ((ElasticsearchSecurityException) t).status() == RestStatus.UNAUTHORIZED;
+            ese = (ElasticsearchSecurityException) t;
+            if (ese.getHeader("WWW-Authenticate") != null && ese.getHeader("WWW-Authenticate").isEmpty() == false) {
+                /**
+                 * If 'WWW-Authenticate' header is present with 'Negotiate ' then do not
+                 * replace. In case of kerberos spnego mechanism, we use
+                 * 'WWW-Authenticate' header value to communicate outToken to peer.
+                 */
+                containsNegotiateWithToken =
+                        ese.getHeader("WWW-Authenticate").stream()
+                                .anyMatch(s -> s != null && s.regionMatches(true, 0, "Negotiate ", 0, "Negotiate ".length()));
+            } else {
+                containsNegotiateWithToken = false;
+            }
+        } else {
+            ese = authenticationError(message, t, args);
+            containsNegotiateWithToken = false;
+        }
+        defaultFailureResponseHeaders.entrySet().stream().forEach((e) -> {
+            if (containsNegotiateWithToken && e.getKey().equalsIgnoreCase("WWW-Authenticate")) {
+                return;
+            }
+            // If it is already present then it will replace the existing header.
+            ese.addHeader(e.getKey(), e.getValue());
+        });
+        return ese;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java
index 3e92be2ef90..2c63ca95eb9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java
@@ -8,9 +8,12 @@ package org.elasticsearch.xpack.core.security.authc;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.security.user.User;
 
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -56,6 +59,18 @@ public abstract class Realm implements Comparable<Realm> {
         return config.order;
     }
 
+    /**
+     * Each realm can define response headers to be sent on failure.
+     * <p>
+     * By default it adds 'WWW-Authenticate' header with auth scheme 'Basic'.
+     *
+     * @return Map of authentication failure response headers.
+     */
+    public Map<String, List<String>> getAuthenticationFailureHeaders() {
+        return Collections.singletonMap("WWW-Authenticate",
+                Collections.singletonList("Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""));
+    }
+
     @Override
     public int compareTo(Realm other) {
         int result = Integer.compare(config.order, other.config.order);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/kerberos/KerberosRealmSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/kerberos/KerberosRealmSettings.java
new file mode 100644
index 00000000000..7524ef08c1e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/kerberos/KerberosRealmSettings.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authc.kerberos;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.set.Sets;
+
+import java.util.Set;
+
+/**
+ * Kerberos Realm settings
+ */
+public final class KerberosRealmSettings {
+    public static final String TYPE = "kerberos";
+
+    /**
+     * Kerberos key tab for Elasticsearch service<br>
+     * Uses single key tab for multiple service accounts.
+     */
+    public static final Setting<String> HTTP_SERVICE_KEYTAB_PATH =
+            Setting.simpleString("keytab.path", Property.NodeScope);
+    public static final Setting<Boolean> SETTING_KRB_DEBUG_ENABLE =
+            Setting.boolSetting("krb.debug", Boolean.FALSE, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_REMOVE_REALM_NAME =
+            Setting.boolSetting("remove_realm_name", Boolean.FALSE, Property.NodeScope);
+
+    // Cache
+    private static final TimeValue DEFAULT_TTL = TimeValue.timeValueMinutes(20);
+    private static final int DEFAULT_MAX_USERS = 100_000; // 100k users
+    public static final Setting<TimeValue> CACHE_TTL_SETTING = Setting.timeSetting("cache.ttl", DEFAULT_TTL, Setting.Property.NodeScope);
+    public static final Setting<Integer> CACHE_MAX_USERS_SETTING =
+            Setting.intSetting("cache.max_users", DEFAULT_MAX_USERS, Property.NodeScope);
+
+    private KerberosRealmSettings() {
+    }
+
+    /**
+     * @return the valid set of {@link Setting}s for a {@value #TYPE} realm
+     */
+    public static Set<Setting<?>> getSettings() {
+        return Sets.newHashSet(HTTP_SERVICE_KEYTAB_PATH, CACHE_TTL_SETTING, CACHE_MAX_USERS_SETTING, SETTING_KRB_DEBUG_ENABLE,
+                SETTING_REMOVE_REALM_NAME);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java
new file mode 100644
index 00000000000..2598461c372
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authc;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackField;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class DefaultAuthenticationFailureHandlerTests extends ESTestCase {
+
+    public void testAuthenticationRequired() {
+        final boolean testDefault = randomBoolean();
+        final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"";
+        final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\"";
+        final DefaultAuthenticationFailureHandler failuerHandler;
+        if (testDefault) {
+            failuerHandler = new DefaultAuthenticationFailureHandler();
+        } else {
+            final Map<String, List<String>> failureResponeHeaders = new HashMap<>();
+            failureResponeHeaders.put("WWW-Authenticate", Arrays.asList(basicAuthScheme, bearerAuthScheme));
+            failuerHandler = new DefaultAuthenticationFailureHandler(failureResponeHeaders);
+        }
+        assertThat(failuerHandler, is(notNullValue()));
+        final ElasticsearchSecurityException ese =
+                failuerHandler.authenticationRequired("someaction", new ThreadContext(Settings.builder().build()));
+        assertThat(ese, is(notNullValue()));
+        assertThat(ese.getMessage(), equalTo("action [someaction] requires authentication"));
+        assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue()));
+        if (testDefault) {
+            assertWWWAuthenticateWithSchemes(ese, basicAuthScheme);
+        } else {
+            assertWWWAuthenticateWithSchemes(ese, basicAuthScheme, bearerAuthScheme);
+        }
+    }
+
+    public void testExceptionProcessingRequest() {
+        final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"";
+        final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\"";
+        final String negotiateAuthScheme = randomFrom("Negotiate", "Negotiate Ijoijksdk");
+        final Map<String, List<String>> failureResponeHeaders = new HashMap<>();
+        failureResponeHeaders.put("WWW-Authenticate", Arrays.asList(basicAuthScheme, bearerAuthScheme, negotiateAuthScheme));
+        final DefaultAuthenticationFailureHandler failuerHandler = new DefaultAuthenticationFailureHandler(failureResponeHeaders);
+
+        assertThat(failuerHandler, is(notNullValue()));
+        final boolean causeIsElasticsearchSecurityException = randomBoolean();
+        final boolean causeIsEseAndUnauthorized = causeIsElasticsearchSecurityException && randomBoolean();
+        final ElasticsearchSecurityException eseCause = (causeIsEseAndUnauthorized)
+                ? new ElasticsearchSecurityException("unauthorized", RestStatus.UNAUTHORIZED, null, (Object[]) null)
+                : new ElasticsearchSecurityException("different error", RestStatus.BAD_REQUEST, null, (Object[]) null);
+        final Exception cause = causeIsElasticsearchSecurityException ? eseCause : new Exception("other error");
+        final boolean withAuthenticateHeader = randomBoolean();
+        final String selectedScheme = randomFrom(bearerAuthScheme, basicAuthScheme, negotiateAuthScheme);
+        if (withAuthenticateHeader) {
+            eseCause.addHeader("WWW-Authenticate", Collections.singletonList(selectedScheme));
+        }
+
+        if (causeIsElasticsearchSecurityException) {
+            if (causeIsEseAndUnauthorized) {
+                final ElasticsearchSecurityException ese = failuerHandler.exceptionProcessingRequest(Mockito.mock(RestRequest.class), cause,
+                        new ThreadContext(Settings.builder().build()));
+                assertThat(ese, is(notNullValue()));
+                assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue()));
+                assertThat(ese, is(sameInstance(cause)));
+                if (withAuthenticateHeader == false) {
+                    assertWWWAuthenticateWithSchemes(ese, basicAuthScheme, bearerAuthScheme, negotiateAuthScheme);
+                } else {
+                    if (selectedScheme.contains("Negotiate ")) {
+                        assertWWWAuthenticateWithSchemes(ese, selectedScheme);
+                    } else {
+                        assertWWWAuthenticateWithSchemes(ese, basicAuthScheme, bearerAuthScheme, negotiateAuthScheme);
+                    }
+                }
+                assertThat(ese.getMessage(), equalTo("unauthorized"));
+            } else {
+                expectThrows(AssertionError.class, () -> failuerHandler.exceptionProcessingRequest(Mockito.mock(RestRequest.class), cause,
+                        new ThreadContext(Settings.builder().build())));
+            }
+        } else {
+            final ElasticsearchSecurityException ese = failuerHandler.exceptionProcessingRequest(Mockito.mock(RestRequest.class), cause,
+                    new ThreadContext(Settings.builder().build()));
+            assertThat(ese, is(notNullValue()));
+            assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue()));
+            assertThat(ese.getMessage(), equalTo("error attempting to authenticate request"));
+            assertWWWAuthenticateWithSchemes(ese, basicAuthScheme, bearerAuthScheme, negotiateAuthScheme);
+        }
+
+    }
+
+    private void assertWWWAuthenticateWithSchemes(final ElasticsearchSecurityException ese, final String... schemes) {
+        assertThat(ese.getHeader("WWW-Authenticate").size(), is(schemes.length));
+        assertThat(ese.getHeader("WWW-Authenticate"), contains(schemes));
+    }
+}
diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle
index 854b0165954..4b17bd47812 100644
--- a/x-pack/plugin/security/build.gradle
+++ b/x-pack/plugin/security/build.gradle
@@ -52,12 +52,73 @@ dependencies {
     compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
     compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
     compile "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
-    compile 'com.google.guava:guava:19.0' 
+    compile 'com.google.guava:guava:19.0'
 
     testCompile 'org.elasticsearch:securemock:1.2'
     testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
     //testCompile "org.yaml:snakeyaml:${versions.snakeyaml}"
 
+    // Test dependencies for Kerberos (MiniKdc)
+    testCompile('commons-io:commons-io:2.5')
+    testCompile('org.apache.kerby:kerb-simplekdc:1.1.1')
+    testCompile('org.apache.kerby:kerb-client:1.1.1')
+    testCompile('org.apache.kerby:kerby-config:1.1.1')
+    testCompile('org.apache.kerby:kerb-core:1.1.1')
+    testCompile('org.apache.kerby:kerby-pkix:1.1.1')
+    testCompile('org.apache.kerby:kerby-asn1:1.1.1')
+    testCompile('org.apache.kerby:kerby-util:1.1.1')
+    testCompile('org.apache.kerby:kerb-common:1.1.1')
+    testCompile('org.apache.kerby:kerb-crypto:1.1.1')
+    testCompile('org.apache.kerby:kerb-util:1.1.1')
+    testCompile('org.apache.kerby:token-provider:1.1.1')
+    testCompile('com.nimbusds:nimbus-jose-jwt:4.41.2')
+    testCompile('net.jcip:jcip-annotations:1.0')
+    testCompile('org.apache.kerby:kerb-admin:1.1.1')
+    testCompile('org.apache.kerby:kerb-server:1.1.1')
+    testCompile('org.apache.kerby:kerb-identity:1.1.1')
+    testCompile('org.apache.kerby:kerby-xdr:1.1.1')
+
+    // LDAP backend support for SimpleKdcServer
+    testCompile('org.apache.kerby:kerby-backend:1.1.1')
+    testCompile('org.apache.kerby:ldap-backend:1.1.1')
+    testCompile('org.apache.kerby:kerb-identity:1.1.1')
+    testCompile('org.apache.directory.api:api-ldap-client-api:1.0.0')
+    testCompile('org.apache.directory.api:api-ldap-schema-data:1.0.0')
+    testCompile('org.apache.directory.api:api-ldap-codec-core:1.0.0')
+    testCompile('org.apache.directory.api:api-ldap-extras-aci:1.0.0')
+    testCompile('org.apache.directory.api:api-ldap-extras-codec:1.0.0')
+    testCompile('org.apache.directory.api:api-ldap-extras-codec-api:1.0.0')
+    testCompile('commons-pool:commons-pool:1.6')
+    testCompile('commons-collections:commons-collections:3.2')
+    testCompile('org.apache.mina:mina-core:2.0.17')
+    testCompile('org.apache.directory.api:api-util:1.0.1')
+    testCompile('org.apache.directory.api:api-i18n:1.0.1')
+    testCompile('org.apache.directory.api:api-ldap-model:1.0.1')
+    testCompile('org.apache.directory.api:api-asn1-api:1.0.1')
+    testCompile('org.apache.directory.api:api-asn1-ber:1.0.1')
+    testCompile('org.apache.servicemix.bundles:org.apache.servicemix.bundles.antlr:2.7.7_5')
+    testCompile('org.apache.directory.server:apacheds-core-api:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-i18n:2.0.0-M24')
+    testCompile('org.apache.directory.api:api-ldap-extras-util:1.0.0')
+    testCompile('net.sf.ehcache:ehcache:2.10.4')
+    testCompile('org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-protocol-ldap:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-protocol-shared:2.0.0-M24')
+    testCompile('org.apache.directory.jdbm:apacheds-jdbm1:2.0.0-M3')
+    testCompile('org.apache.directory.server:apacheds-jdbm-partition:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-xdbm-partition:2.0.0-M24')
+    testCompile('org.apache.directory.api:api-ldap-extras-sp:1.0.0')
+    testCompile('org.apache.directory.server:apacheds-test-framework:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-core-annotations:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-ldif-partition:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-mavibot-partition:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-protocol-kerberos:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-server-annotations:2.0.0-M24')
+    testCompile('org.apache.directory.api:api-ldap-codec-standalone:1.0.0')
+    testCompile('org.apache.directory.api:api-ldap-net-mina:1.0.0')
+    testCompile('org.apache.directory.server:ldap-client-test:2.0.0-M24')
+    testCompile('org.apache.directory.server:apacheds-interceptor-kerberos:2.0.0-M24')
+    testCompile('org.apache.directory.mavibot:mavibot:1.0.0-M8')
 }
 
 compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 98b5df3edc5..e483ad235c5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -77,6 +77,7 @@ import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -168,6 +169,7 @@ import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosRealmBootstrapCheck;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener;
@@ -291,7 +293,8 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 new TokenSSLBootstrapCheck(),
                 new PkiRealmBootstrapCheck(getSslService()),
                 new TLSLicenseBootstrapCheck(),
-                new PasswordHashingAlgorithmBootstrapCheck()));
+                new PasswordHashingAlgorithmBootstrapCheck(),
+                new KerberosRealmBootstrapCheck(env)));
             checks.addAll(InternalRealms.getBootstrapChecks(settings, env));
             this.bootstrapChecks = Collections.unmodifiableList(checks);
         } else {
@@ -432,23 +435,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
 
         securityIndex.get().addIndexStateListener(nativeRoleMappingStore::onSecurityIndexStateChange);
 
-        AuthenticationFailureHandler failureHandler = null;
-        String extensionName = null;
-        for (SecurityExtension extension : securityExtensions) {
-            AuthenticationFailureHandler extensionFailureHandler = extension.getAuthenticationFailureHandler();
-            if (extensionFailureHandler != null && failureHandler != null) {
-                throw new IllegalStateException("Extensions [" + extensionName + "] and [" + extension.toString() + "] " +
-                    "both set an authentication failure handler");
-            }
-            failureHandler = extensionFailureHandler;
-            extensionName = extension.toString();
-        }
-        if (failureHandler == null) {
-            logger.debug("Using default authentication failure handler");
-            failureHandler = new DefaultAuthenticationFailureHandler();
-        } else {
-            logger.debug("Using authentication failure handler from extension [" + extensionName + "]");
-        }
+        final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms);
 
         authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool,
                 anonymousUser, tokenService));
@@ -498,6 +485,45 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         return components;
     }
 
+    private AuthenticationFailureHandler createAuthenticationFailureHandler(final Realms realms) {
+        AuthenticationFailureHandler failureHandler = null;
+        String extensionName = null;
+        for (SecurityExtension extension : securityExtensions) {
+            AuthenticationFailureHandler extensionFailureHandler = extension.getAuthenticationFailureHandler();
+            if (extensionFailureHandler != null && failureHandler != null) {
+                throw new IllegalStateException("Extensions [" + extensionName + "] and [" + extension.toString() + "] "
+                        + "both set an authentication failure handler");
+            }
+            failureHandler = extensionFailureHandler;
+            extensionName = extension.toString();
+        }
+        if (failureHandler == null) {
+            logger.debug("Using default authentication failure handler");
+            final Map<String, List<String>> defaultFailureResponseHeaders = new HashMap<>();
+            realms.asList().stream().forEach((realm) -> {
+                Map<String, List<String>> realmFailureHeaders = realm.getAuthenticationFailureHeaders();
+                realmFailureHeaders.entrySet().stream().forEach((e) -> {
+                    String key = e.getKey();
+                    e.getValue().stream()
+                            .filter(v -> defaultFailureResponseHeaders.computeIfAbsent(key, x -> new ArrayList<>()).contains(v) == false)
+                            .forEach(v -> defaultFailureResponseHeaders.get(key).add(v));
+                });
+            });
+
+            if (TokenService.isTokenServiceEnabled(settings)) {
+                String bearerScheme = "Bearer realm=\"" + XPackField.SECURITY + "\"";
+                if (defaultFailureResponseHeaders.computeIfAbsent("WWW-Authenticate", x -> new ArrayList<>())
+                        .contains(bearerScheme) == false) {
+                    defaultFailureResponseHeaders.get("WWW-Authenticate").add(bearerScheme);
+                }
+            }
+            failureHandler = new DefaultAuthenticationFailureHandler(defaultFailureResponseHeaders);
+        } else {
+            logger.debug("Using authentication failure handler from extension [" + extensionName + "]");
+        }
+        return failureHandler;
+    }
+
     @Override
     public Settings additionalSettings() {
         return additionalSettings(settings, enabled, transportClientMode);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
index 3898e34b7a4..85084da8464 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
@@ -271,7 +271,9 @@ public class AuthenticationService extends AbstractComponent {
                                 if (result.getStatus() == AuthenticationResult.Status.TERMINATE) {
                                     logger.info("Authentication of [{}] was terminated by realm [{}] - {}",
                                             authenticationToken.principal(), realm.name(), result.getMessage());
-                                    userListener.onFailure(Exceptions.authenticationError(result.getMessage(), result.getException()));
+                                    Exception e = (result.getException() != null) ? result.getException()
+                                            : Exceptions.authenticationError(result.getMessage());
+                                    userListener.onFailure(e);
                                 } else {
                                     if (result.getMessage() != null) {
                                         messages.put(realm, new Tuple<>(result.getMessage(), result.getException()));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java
index d8d0d26f99e..d568a052a5e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java
@@ -16,6 +16,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
@@ -24,6 +25,7 @@ import org.elasticsearch.xpack.security.authc.esnative.NativeRealm;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.file.FileRealm;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosRealm;
 import org.elasticsearch.xpack.security.authc.ldap.LdapRealm;
 import org.elasticsearch.xpack.security.authc.pki.PkiRealm;
 import org.elasticsearch.xpack.security.authc.saml.SamlRealm;
@@ -32,10 +34,8 @@ import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingSt
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -50,17 +50,16 @@ public final class InternalRealms {
     /**
      * The list of all <em>internal</em> realm types, excluding {@link ReservedRealm#TYPE}.
      */
-    private static final Set<String> XPACK_TYPES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
-            NativeRealmSettings.TYPE, FileRealmSettings.TYPE, LdapRealmSettings.AD_TYPE, LdapRealmSettings.LDAP_TYPE, PkiRealmSettings.TYPE,
-            SamlRealmSettings.TYPE
-    )));
+    private static final Set<String> XPACK_TYPES = Collections
+            .unmodifiableSet(Sets.newHashSet(NativeRealmSettings.TYPE, FileRealmSettings.TYPE, LdapRealmSettings.AD_TYPE,
+                    LdapRealmSettings.LDAP_TYPE, PkiRealmSettings.TYPE, SamlRealmSettings.TYPE, KerberosRealmSettings.TYPE));
 
     /**
      * The list of all standard realm types, which are those provided by x-pack and do not have extensive
      * interaction with third party sources
      */
-    private static final Set<String> STANDARD_TYPES =
-            Collections.unmodifiableSet(Sets.difference(XPACK_TYPES, Collections.singleton(SamlRealmSettings.TYPE)));
+    private static final Set<String> STANDARD_TYPES = Collections.unmodifiableSet(Sets.newHashSet(NativeRealmSettings.TYPE,
+            FileRealmSettings.TYPE, LdapRealmSettings.AD_TYPE, LdapRealmSettings.LDAP_TYPE, PkiRealmSettings.TYPE));
 
     /**
      * Determines whether <code>type</code> is an internal realm-type that is provided by x-pack,
@@ -105,6 +104,7 @@ public final class InternalRealms {
                 sslService, resourceWatcherService, nativeRoleMappingStore, threadPool));
         map.put(PkiRealmSettings.TYPE, config -> new PkiRealm(config, resourceWatcherService, nativeRoleMappingStore));
         map.put(SamlRealmSettings.TYPE, config -> SamlRealm.create(config, sslService, resourceWatcherService, nativeRoleMappingStore));
+        map.put(KerberosRealmSettings.TYPE, config -> new KerberosRealm(config, nativeRoleMappingStore, threadPool));
         return Collections.unmodifiableMap(map);
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationToken.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationToken.java
new file mode 100644
index 00000000000..1a330bd2ddd
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationToken.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+
+import java.util.Arrays;
+import java.util.Base64;
+
+/**
+ * This class represents an AuthenticationToken for Kerberos authentication
+ * using SPNEGO. The token stores base 64 decoded token bytes, extracted from
+ * the Authorization header with auth scheme 'Negotiate'.
+ * <p>
+ * Example Authorization header "Authorization: Negotiate
+ * YIIChgYGKwYBBQUCoII..."
+ * <p>
+ * If there is any error handling during extraction of 'Negotiate' header then
+ * it throws {@link ElasticsearchSecurityException} with
+ * {@link RestStatus#UNAUTHORIZED} and header 'WWW-Authenticate: Negotiate'
+ */
+public final class KerberosAuthenticationToken implements AuthenticationToken {
+
+    public static final String WWW_AUTHENTICATE = "WWW-Authenticate";
+    public static final String AUTH_HEADER = "Authorization";
+    public static final String NEGOTIATE_SCHEME_NAME = "Negotiate";
+    public static final String NEGOTIATE_AUTH_HEADER_PREFIX = NEGOTIATE_SCHEME_NAME + " ";
+
+    // authorization scheme check is case-insensitive
+    private static final boolean IGNORE_CASE_AUTH_HEADER_MATCH = true;
+
+    private final byte[] decodedToken;
+
+    public KerberosAuthenticationToken(final byte[] decodedToken) {
+        this.decodedToken = decodedToken;
+    }
+
+    /**
+     * Extract token from authorization header and if it is valid
+     * {@value #NEGOTIATE_AUTH_HEADER_PREFIX} then returns
+     * {@link KerberosAuthenticationToken}
+     *
+     * @param authorizationHeader Authorization header from request
+     * @return returns {@code null} if {@link #AUTH_HEADER} is empty or does not
+     *         start with {@value #NEGOTIATE_AUTH_HEADER_PREFIX} else returns valid
+     *         {@link KerberosAuthenticationToken}
+     * @throws ElasticsearchSecurityException when negotiate header is invalid.
+     */
+    public static KerberosAuthenticationToken extractToken(final String authorizationHeader) {
+        if (Strings.isNullOrEmpty(authorizationHeader)) {
+            return null;
+        }
+        if (authorizationHeader.regionMatches(IGNORE_CASE_AUTH_HEADER_MATCH, 0, NEGOTIATE_AUTH_HEADER_PREFIX, 0,
+                NEGOTIATE_AUTH_HEADER_PREFIX.length()) == false) {
+            return null;
+        }
+
+        final String base64EncodedToken = authorizationHeader.substring(NEGOTIATE_AUTH_HEADER_PREFIX.length()).trim();
+        if (Strings.isEmpty(base64EncodedToken)) {
+            throw unauthorized("invalid negotiate authentication header value, expected base64 encoded token but value is empty", null);
+        }
+
+        byte[] decodedKerberosTicket = null;
+        try {
+            decodedKerberosTicket = Base64.getDecoder().decode(base64EncodedToken);
+        } catch (IllegalArgumentException iae) {
+            throw unauthorized("invalid negotiate authentication header value, could not decode base64 token {}", iae, base64EncodedToken);
+        }
+
+        return new KerberosAuthenticationToken(decodedKerberosTicket);
+    }
+
+    @Override
+    public String principal() {
+        return "<Kerberos Token>";
+    }
+
+    @Override
+    public Object credentials() {
+        return decodedToken;
+    }
+
+    @Override
+    public void clearCredentials() {
+        Arrays.fill(decodedToken, (byte) 0);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(decodedToken);
+    }
+
+    @Override
+    public boolean equals(final Object other) {
+        if (this == other)
+            return true;
+        if (other == null)
+            return false;
+        if (getClass() != other.getClass())
+            return false;
+        final KerberosAuthenticationToken otherKerbToken = (KerberosAuthenticationToken) other;
+        return Arrays.equals(otherKerbToken.decodedToken, this.decodedToken);
+    }
+
+    /**
+     * Creates {@link ElasticsearchSecurityException} with
+     * {@link RestStatus#UNAUTHORIZED} and cause. This also populates
+     * 'WWW-Authenticate' header with value as 'Negotiate' scheme.
+     *
+     * @param message the detail message
+     * @param cause nested exception
+     * @param args the arguments for the message
+     * @return instance of {@link ElasticsearchSecurityException}
+     */
+    static ElasticsearchSecurityException unauthorized(final String message, final Throwable cause, final Object... args) {
+        ElasticsearchSecurityException ese = new ElasticsearchSecurityException(message, RestStatus.UNAUTHORIZED, cause, args);
+        ese.addHeader(WWW_AUTHENTICATE, NEGOTIATE_SCHEME_NAME);
+        return ese;
+    }
+
+    /**
+     * Sets 'WWW-Authenticate' header if outToken is not null on passed instance of
+     * {@link ElasticsearchSecurityException} and returns the instance. <br>
+     * If outToken is provided and is not {@code null} or empty, then that is
+     * appended to 'Negotiate ' and is used as header value for header
+     * 'WWW-Authenticate' sent to the peer in the form 'Negotiate oYH1MIHyoAMK...'.
+     * This is required by client for GSS negotiation to continue further.
+     *
+     * @param ese instance of {@link ElasticsearchSecurityException} with status
+     *            {@link RestStatus#UNAUTHORIZED}
+     * @param outToken if non {@code null} and not empty then this will be the value
+     *            sent to the peer.
+     * @return instance of {@link ElasticsearchSecurityException} with
+     *         'WWW-Authenticate' header populated.
+     */
+    static ElasticsearchSecurityException unauthorizedWithOutputToken(final ElasticsearchSecurityException ese, final String outToken) {
+        assert ese.status() == RestStatus.UNAUTHORIZED;
+        if (Strings.hasText(outToken)) {
+            ese.addHeader(WWW_AUTHENTICATE, NEGOTIATE_AUTH_HEADER_PREFIX + outToken);
+        }
+        return ese;
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
new file mode 100644
index 00000000000..b4a8b6aabf0
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
@@ -0,0 +1,214 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+import org.elasticsearch.xpack.core.security.authc.Realm;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.support.CachingRealm;
+import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
+import org.ietf.jgss.GSSException;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import javax.security.auth.login.LoginException;
+
+import static org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken.AUTH_HEADER;
+import static org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX;
+import static org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken.NEGOTIATE_SCHEME_NAME;
+import static org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken.WWW_AUTHENTICATE;
+import static org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken.unauthorized;
+import static org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken.unauthorizedWithOutputToken;
+
+/**
+ * This class provides support for Kerberos authentication using spnego
+ * mechanism.
+ * <p>
+ * It provides support to extract kerberos ticket using
+ * {@link KerberosAuthenticationToken#extractToken(String)} to build
+ * {@link KerberosAuthenticationToken} and then authenticating user when
+ * {@link KerberosTicketValidator} validates the ticket.
+ * <p>
+ * On successful authentication, it will build {@link User} object populated
+ * with roles and will return {@link AuthenticationResult} with user object. On
+ * authentication failure, it will return {@link AuthenticationResult} with
+ * status to terminate authentication process.
+ */
+public final class KerberosRealm extends Realm implements CachingRealm {
+
+    private final Cache<String, User> userPrincipalNameToUserCache;
+    private final NativeRoleMappingStore userRoleMapper;
+    private final KerberosTicketValidator kerberosTicketValidator;
+    private final ThreadPool threadPool;
+    private final Path keytabPath;
+    private final boolean enableKerberosDebug;
+    private final boolean removeRealmName;
+
+    public KerberosRealm(final RealmConfig config, final NativeRoleMappingStore nativeRoleMappingStore, final ThreadPool threadPool) {
+        this(config, nativeRoleMappingStore, new KerberosTicketValidator(), threadPool, null);
+    }
+
+    // pkg scoped for testing
+    KerberosRealm(final RealmConfig config, final NativeRoleMappingStore nativeRoleMappingStore,
+            final KerberosTicketValidator kerberosTicketValidator, final ThreadPool threadPool,
+            final Cache<String, User> userPrincipalNameToUserCache) {
+        super(KerberosRealmSettings.TYPE, config);
+        this.userRoleMapper = nativeRoleMappingStore;
+        this.userRoleMapper.refreshRealmOnChange(this);
+        final TimeValue ttl = KerberosRealmSettings.CACHE_TTL_SETTING.get(config.settings());
+        if (ttl.getNanos() > 0) {
+            this.userPrincipalNameToUserCache = (userPrincipalNameToUserCache == null)
+                    ? CacheBuilder.<String, User>builder()
+                            .setExpireAfterWrite(KerberosRealmSettings.CACHE_TTL_SETTING.get(config.settings()))
+                            .setMaximumWeight(KerberosRealmSettings.CACHE_MAX_USERS_SETTING.get(config.settings())).build()
+                    : userPrincipalNameToUserCache;
+        } else {
+            this.userPrincipalNameToUserCache = null;
+        }
+        this.kerberosTicketValidator = kerberosTicketValidator;
+        this.threadPool = threadPool;
+        this.keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+        this.enableKerberosDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
+        this.removeRealmName = KerberosRealmSettings.SETTING_REMOVE_REALM_NAME.get(config.settings());
+    }
+
+    @Override
+    public Map<String, List<String>> getAuthenticationFailureHeaders() {
+        return Collections.singletonMap(WWW_AUTHENTICATE, Collections.singletonList(NEGOTIATE_SCHEME_NAME));
+    }
+
+    @Override
+    public void expire(final String username) {
+        if (userPrincipalNameToUserCache != null) {
+            userPrincipalNameToUserCache.invalidate(username);
+        }
+    }
+
+    @Override
+    public void expireAll() {
+        if (userPrincipalNameToUserCache != null) {
+            userPrincipalNameToUserCache.invalidateAll();
+        }
+    }
+
+    @Override
+    public boolean supports(final AuthenticationToken token) {
+        return token instanceof KerberosAuthenticationToken;
+    }
+
+    @Override
+    public AuthenticationToken token(final ThreadContext context) {
+        return KerberosAuthenticationToken.extractToken(context.getHeader(AUTH_HEADER));
+    }
+
+    @Override
+    public void authenticate(final AuthenticationToken token, final ActionListener<AuthenticationResult> listener) {
+        assert token instanceof KerberosAuthenticationToken;
+        final KerberosAuthenticationToken kerbAuthnToken = (KerberosAuthenticationToken) token;
+        kerberosTicketValidator.validateTicket((byte[]) kerbAuthnToken.credentials(), keytabPath, enableKerberosDebug,
+                ActionListener.wrap(userPrincipalNameOutToken -> {
+                    if (userPrincipalNameOutToken.v1() != null) {
+                        final String username = maybeRemoveRealmName(userPrincipalNameOutToken.v1());
+                        buildUser(username, userPrincipalNameOutToken.v2(), listener);
+                    } else {
+                        /**
+                         * This is when security context could not be established may be due to ongoing
+                         * negotiation and requires token to be sent back to peer for continuing
+                         * further. We are terminating the authentication process as this is spengo
+                         * negotiation and no other realm can handle this. We can have only one Kerberos
+                         * realm in the system so terminating with RestStatus Unauthorized (401) and
+                         * with 'WWW-Authenticate' header populated with value with token in the form
+                         * 'Negotiate oYH1MIHyoAMK...'
+                         */
+                        String errorMessage = "failed to authenticate user, gss context negotiation not complete";
+                        ElasticsearchSecurityException ese = unauthorized(errorMessage, null);
+                        ese = unauthorizedWithOutputToken(ese, userPrincipalNameOutToken.v2());
+                        listener.onResponse(AuthenticationResult.terminate(errorMessage, ese));
+                    }
+                }, e -> handleException(e, listener)));
+    }
+
+    /**
+     * Usually principal names are in the form 'user/instance@REALM'. This method
+     * removes '@REALM' part from the principal name if
+     * {@link KerberosRealmSettings#SETTING_REMOVE_REALM_NAME} is {@code true} else
+     * will return the input string.
+     *
+     * @param principalName user principal name
+     * @return username after removal of realm
+     */
+    protected String maybeRemoveRealmName(final String principalName) {
+        if (this.removeRealmName) {
+            int foundAtIndex = principalName.indexOf('@');
+            if (foundAtIndex > 0) {
+                return principalName.substring(0, foundAtIndex);
+            }
+        }
+        return principalName;
+    }
+
+    private void handleException(Exception e, final ActionListener<AuthenticationResult> listener) {
+        if (e instanceof LoginException) {
+            listener.onResponse(AuthenticationResult.terminate("failed to authenticate user, service login failure",
+                    unauthorized(e.getLocalizedMessage(), e)));
+        } else if (e instanceof GSSException) {
+            listener.onResponse(AuthenticationResult.terminate("failed to authenticate user, gss context negotiation failure",
+                    unauthorized(e.getLocalizedMessage(), e)));
+        } else {
+            listener.onFailure(e);
+        }
+    }
+
+    private void buildUser(final String username, final String outToken, final ActionListener<AuthenticationResult> listener) {
+        // if outToken is present then it needs to be communicated with peer, add it to
+        // response header in thread context.
+        if (Strings.hasText(outToken)) {
+            threadPool.getThreadContext().addResponseHeader(WWW_AUTHENTICATE, NEGOTIATE_AUTH_HEADER_PREFIX + outToken);
+        }
+        final User user = (userPrincipalNameToUserCache != null) ? userPrincipalNameToUserCache.get(username) : null;
+        if (user != null) {
+            /**
+             * TODO: bizybot If authorizing realms configured, resolve user from those
+             * realms and then return.
+             */
+            listener.onResponse(AuthenticationResult.success(user));
+        } else {
+            /**
+             * TODO: bizybot If authorizing realms configured, resolve user from those
+             * realms, cache it and then return.
+             */
+            final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(username, null, Collections.emptySet(), null, this.config);
+            userRoleMapper.resolveRoles(userData, ActionListener.wrap(roles -> {
+                final User computedUser = new User(username, roles.toArray(new String[roles.size()]), null, null, null, true);
+                if (userPrincipalNameToUserCache != null) {
+                    userPrincipalNameToUserCache.put(username, computedUser);
+                }
+                listener.onResponse(AuthenticationResult.success(computedUser));
+            }, listener::onFailure));
+        }
+    }
+
+    @Override
+    public void lookupUser(final String username, final ActionListener<User> listener) {
+        listener.onResponse(null);
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java
new file mode 100644
index 00000000000..bab899a8664
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.security.authc.RealmSettings;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * This class is used to perform bootstrap checks for kerberos realm.
+ * <p>
+ * We use service keytabs for validating incoming kerberos tickets and is a
+ * required configuration. Due to JVM wide system properties for Kerberos we
+ * cannot support multiple Kerberos realms. This class adds checks for node to
+ * fail if service keytab does not exist or multiple kerberos realms have been
+ * configured.
+ */
+public class KerberosRealmBootstrapCheck implements BootstrapCheck {
+    private final Environment env;
+
+    public KerberosRealmBootstrapCheck(final Environment env) {
+        this.env = env;
+    }
+
+    @Override
+    public BootstrapCheckResult check(final BootstrapContext context) {
+        final Map<String, Settings> realmsSettings = RealmSettings.getRealmSettings(context.settings);
+        boolean isKerberosRealmConfigured = false;
+        for (final Entry<String, Settings> entry : realmsSettings.entrySet()) {
+            final String name = entry.getKey();
+            final Settings realmSettings = entry.getValue();
+            final String type = realmSettings.get("type");
+            if (Strings.hasText(type) == false) {
+                return BootstrapCheckResult.failure("missing realm type for [" + name + "] realm");
+            }
+            if (KerberosRealmSettings.TYPE.equals(type)) {
+                if (isKerberosRealmConfigured) {
+                    return BootstrapCheckResult.failure(
+                            "multiple [" + type + "] realms are configured. [" + type + "] can only have one such realm configured");
+                }
+                isKerberosRealmConfigured = true;
+
+                final Path keytabPath = env.configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(realmSettings));
+                if (Files.exists(keytabPath) == false) {
+                    return BootstrapCheckResult.failure("configured service key tab file [" + keytabPath + "] does not exist");
+                }
+            }
+        }
+        return BootstrapCheckResult.success();
+    }
+
+    @Override
+    public boolean alwaysEnforce() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidator.java
new file mode 100644
index 00000000000..a63d90178dc
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidator.java
@@ -0,0 +1,273 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.ietf.jgss.GSSContext;
+import org.ietf.jgss.GSSCredential;
+import org.ietf.jgss.GSSException;
+import org.ietf.jgss.GSSManager;
+import org.ietf.jgss.Oid;
+
+import java.nio.file.Path;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.security.auth.Subject;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
+
+/**
+ * Utility class that validates kerberos ticket for peer authentication.
+ * <p>
+ * This class takes care of login by ES service credentials using keytab,
+ * GSSContext establishment, and then validating the incoming token.
+ * <p>
+ * It may respond with token which needs to be communicated with the peer.
+ */
+public class KerberosTicketValidator {
+    static final Oid SPNEGO_OID = getSpnegoOid();
+
+    private static Oid getSpnegoOid() {
+        Oid oid = null;
+        try {
+            oid = new Oid("1.3.6.1.5.5.2");
+        } catch (GSSException gsse) {
+            throw ExceptionsHelper.convertToRuntime(gsse);
+        }
+        return oid;
+    }
+
+    private static final Logger LOGGER = ESLoggerFactory.getLogger(KerberosTicketValidator.class);
+
+    private static final String KEY_TAB_CONF_NAME = "KeytabConf";
+    private static final String SUN_KRB5_LOGIN_MODULE = "com.sun.security.auth.module.Krb5LoginModule";
+
+    /**
+     * Validates client kerberos ticket received from the peer.
+     * <p>
+     * First performs service login using keytab, supports multiple principals in
+     * keytab and the principal is selected based on the request.
+     * <p>
+     * The GSS security context establishment state is handled as follows: <br>
+     * If the context is established it will call {@link ActionListener#onResponse}
+     * with a {@link Tuple} of username and outToken for peer reply. <br>
+     * If the context is not established then it will call
+     * {@link ActionListener#onResponse} with a Tuple where username is null but
+     * with a outToken that needs to be sent to peer for further negotiation. <br>
+     * Never calls {@link ActionListener#onResponse} with a {@code null} tuple. <br>
+     * On failure, it will call {@link ActionListener#onFailure(Exception)}
+     *
+     * @param decodedToken base64 decoded kerberos ticket bytes
+     * @param keytabPath Path to Service key tab file containing credentials for ES
+     *            service.
+     * @param krbDebug if {@code true} enables jaas krb5 login module debug logs.
+     */
+    public void validateTicket(final byte[] decodedToken, final Path keytabPath, final boolean krbDebug,
+            final ActionListener<Tuple<String, String>> actionListener) {
+        final GSSManager gssManager = GSSManager.getInstance();
+        GSSContext gssContext = null;
+        LoginContext loginContext = null;
+        try {
+            loginContext = serviceLogin(keytabPath.toString(), krbDebug);
+            GSSCredential serviceCreds = createCredentials(gssManager, loginContext.getSubject());
+            gssContext = gssManager.createContext(serviceCreds);
+            final String base64OutToken = encodeToString(acceptSecContext(decodedToken, gssContext, loginContext.getSubject()));
+            LOGGER.trace("validateTicket isGSSContextEstablished = {}, username = {}, outToken = {}", gssContext.isEstablished(),
+                    gssContext.getSrcName().toString(), base64OutToken);
+            actionListener.onResponse(new Tuple<>(gssContext.isEstablished() ? gssContext.getSrcName().toString() : null, base64OutToken));
+        } catch (GSSException e) {
+            actionListener.onFailure(e);
+        } catch (PrivilegedActionException pve) {
+            if (pve.getCause() instanceof LoginException) {
+                actionListener.onFailure((LoginException) pve.getCause());
+            } else if (pve.getCause() instanceof GSSException) {
+                actionListener.onFailure((GSSException) pve.getCause());
+            } else {
+                actionListener.onFailure(pve.getException());
+            }
+        } finally {
+            privilegedLogoutNoThrow(loginContext);
+            privilegedDisposeNoThrow(gssContext);
+        }
+    }
+
+    /**
+     * Encodes the specified byte array using base64 encoding scheme
+     *
+     * @param outToken byte array to be encoded
+     * @return String containing base64 encoded characters. returns {@code null} if
+     *         outToken is null or empty.
+     */
+    private String encodeToString(final byte[] outToken) {
+        if (outToken != null && outToken.length > 0) {
+            return Base64.getEncoder().encodeToString(outToken);
+        }
+        return null;
+    }
+
+    /**
+     * Handles GSS context establishment. Received token is passed to the GSSContext
+     * on acceptor side and returns with out token that needs to be sent to peer for
+     * further GSS context establishment.
+     * <p>
+     *
+     * @param base64decodedTicket in token generated by peer
+     * @param gssContext instance of acceptor {@link GSSContext}
+     * @param subject authenticated subject
+     * @return a byte[] containing the token to be sent to the peer. null indicates
+     *         that no token is generated.
+     * @throws PrivilegedActionException
+     * @see GSSContext#acceptSecContext(byte[], int, int)
+     */
+    private static byte[] acceptSecContext(final byte[] base64decodedTicket, final GSSContext gssContext, Subject subject)
+            throws PrivilegedActionException {
+        // process token with gss context
+        return doAsWrapper(subject,
+                (PrivilegedExceptionAction<byte[]>) () -> gssContext.acceptSecContext(base64decodedTicket, 0, base64decodedTicket.length));
+    }
+
+    /**
+     * For acquiring SPNEGO mechanism credentials for service based on the subject
+     *
+     * @param gssManager {@link GSSManager}
+     * @param subject logged in {@link Subject}
+     * @return {@link GSSCredential} for particular mechanism
+     * @throws PrivilegedActionException
+     */
+    private static GSSCredential createCredentials(final GSSManager gssManager, final Subject subject) throws PrivilegedActionException {
+        return doAsWrapper(subject, (PrivilegedExceptionAction<GSSCredential>) () -> gssManager.createCredential(null,
+                GSSCredential.DEFAULT_LIFETIME, SPNEGO_OID, GSSCredential.ACCEPT_ONLY));
+    }
+
+    /**
+     * Privileged Wrapper that invokes action with Subject.doAs to perform work as
+     * given subject.
+     *
+     * @param subject {@link Subject} to be used for this work
+     * @param action {@link PrivilegedExceptionAction} action for performing inside
+     *            Subject.doAs
+     * @return the value returned by the PrivilegedExceptionAction's run method
+     * @throws PrivilegedActionException
+     */
+    private static <T> T doAsWrapper(final Subject subject, final PrivilegedExceptionAction<T> action) throws PrivilegedActionException {
+        try {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<T>) () -> Subject.doAs(subject, action));
+        } catch (PrivilegedActionException pae) {
+            if (pae.getCause() instanceof PrivilegedActionException) {
+                throw (PrivilegedActionException) pae.getCause();
+            }
+            throw pae;
+        }
+    }
+
+    /**
+     * Privileged wrapper for closing GSSContext, does not throw exceptions but logs
+     * them as a debug message.
+     *
+     * @param gssContext GSSContext to be disposed.
+     */
+    private static void privilegedDisposeNoThrow(final GSSContext gssContext) {
+        if (gssContext != null) {
+            try {
+                AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                    gssContext.dispose();
+                    return null;
+                });
+            } catch (PrivilegedActionException e) {
+                LOGGER.debug("Could not dispose GSS Context", e.getCause());
+            }
+        }
+    }
+
+    /**
+     * Privileged wrapper for closing LoginContext, does not throw exceptions but
+     * logs them as a debug message.
+     *
+     * @param loginContext LoginContext to be closed
+     */
+    private static void privilegedLogoutNoThrow(final LoginContext loginContext) {
+        if (loginContext != null) {
+            try {
+                AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                    loginContext.logout();
+                    return null;
+                });
+            } catch (PrivilegedActionException e) {
+                LOGGER.debug("Could not close LoginContext", e.getCause());
+            }
+        }
+    }
+
+    /**
+     * Performs authentication using provided keytab
+     *
+     * @param keytabFilePath Keytab file path
+     * @param krbDebug if {@code true} enables jaas krb5 login module debug logs.
+     * @return authenticated {@link LoginContext} instance. Note: This needs to be
+     *         closed using {@link LoginContext#logout()} after usage.
+     * @throws PrivilegedActionException when privileged action threw exception
+     */
+    private static LoginContext serviceLogin(final String keytabFilePath, final boolean krbDebug) throws PrivilegedActionException {
+        return AccessController.doPrivileged((PrivilegedExceptionAction<LoginContext>) () -> {
+            final Subject subject = new Subject(false, Collections.emptySet(), Collections.emptySet(), Collections.emptySet());
+            final Configuration conf = new KeytabJaasConf(keytabFilePath, krbDebug);
+            final LoginContext loginContext = new LoginContext(KEY_TAB_CONF_NAME, subject, null, conf);
+            loginContext.login();
+            return loginContext;
+        });
+    }
+
+    /**
+     * Usually we would have a JAAS configuration file for login configuration. As
+     * we have static configuration except debug flag, we are constructing in
+     * memory. This avoids additional configuration required from the user.
+     * <p>
+     * As we are using this instead of jaas.conf, this requires refresh of
+     * {@link Configuration} and requires appropriate security permissions to do so.
+     */
+    static class KeytabJaasConf extends Configuration {
+        private final String keytabFilePath;
+        private final boolean krbDebug;
+
+        KeytabJaasConf(final String keytabFilePath, final boolean krbDebug) {
+            this.keytabFilePath = keytabFilePath;
+            this.krbDebug = krbDebug;
+        }
+
+        @Override
+        public AppConfigurationEntry[] getAppConfigurationEntry(final String name) {
+            final Map<String, String> options = new HashMap<>();
+            options.put("keyTab", keytabFilePath);
+            /*
+             * As acceptor, we can have multiple SPNs, we do not want to use particular
+             * principal so it uses "*"
+             */
+            options.put("principal", "*");
+            options.put("useKeyTab", Boolean.TRUE.toString());
+            options.put("storeKey", Boolean.TRUE.toString());
+            options.put("doNotPrompt", Boolean.TRUE.toString());
+            options.put("isInitiator", Boolean.FALSE.toString());
+            options.put("debug", Boolean.toString(krbDebug));
+
+            return new AppConfigurationEntry[] { new AppConfigurationEntry(SUN_KRB5_LOGIN_MODULE,
+                    AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, Collections.unmodifiableMap(options)) };
+        }
+
+    }
+}
diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
index 857c2f6e472..8ce72be3ef9 100644
--- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
@@ -10,6 +10,21 @@ grant {
 
   // needed for multiple server implementations used in tests
   permission java.net.SocketPermission "*", "accept,connect";
+
+  // needed for Kerberos login
+  permission javax.security.auth.AuthPermission "modifyPrincipals";
+  permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
+  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosKey * \"*\"", "read";
+  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read";
+  permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosTicket * \"*\"", "read";
+  permission javax.security.auth.AuthPermission "doAs";
+  permission javax.security.auth.kerberos.ServicePermission "*","initiate,accept";
+
+  permission java.util.PropertyPermission "javax.security.auth.useSubjectCredsOnly","write";
+  permission java.util.PropertyPermission "java.security.krb5.conf","write";
+  permission java.util.PropertyPermission "sun.security.krb5.debug","write";
+  permission java.util.PropertyPermission "java.security.debug","write";
+  permission java.util.PropertyPermission "sun.security.spnego.debug","write";
 };
 
 grant codeBase "${codebase.xmlsec-2.0.8.jar}" {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index bb32ed69995..fda87d0340b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -49,6 +49,7 @@ import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportMessage;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
@@ -88,6 +89,7 @@ import static org.elasticsearch.test.SecurityTestsUtils.assertAuthenticationExce
 import static org.elasticsearch.xpack.core.security.support.Exceptions.authenticationError;
 import static org.elasticsearch.xpack.security.authc.TokenServiceTests.mockGetTokenFromId;
 import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -618,6 +620,47 @@ public class AuthenticationServiceTests extends ESTestCase {
         }
     }
 
+    public void testRealmAuthenticateTerminatingAuthenticationProcess() throws Exception {
+        final AuthenticationToken token = mock(AuthenticationToken.class);
+        when(secondRealm.token(threadContext)).thenReturn(token);
+        when(secondRealm.supports(token)).thenReturn(true);
+        final boolean terminateWithNoException = rarely();
+        final boolean throwElasticsearchSecurityException = (terminateWithNoException == false) && randomBoolean();
+        final boolean withAuthenticateHeader = throwElasticsearchSecurityException && randomBoolean();
+        Exception throwE = new Exception("general authentication error");
+        final String basicScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"";
+        String selectedScheme = randomFrom(basicScheme, "Negotiate IOJoj");
+        if (throwElasticsearchSecurityException) {
+            throwE = new ElasticsearchSecurityException("authentication error", RestStatus.UNAUTHORIZED);
+            if (withAuthenticateHeader) {
+                ((ElasticsearchSecurityException) throwE).addHeader("WWW-Authenticate", selectedScheme);
+            }
+        }
+        mockAuthenticate(secondRealm, token, (terminateWithNoException) ? null : throwE, true);
+
+        ElasticsearchSecurityException e =
+                expectThrows(ElasticsearchSecurityException.class, () -> authenticateBlocking("_action", message, null));
+        if (terminateWithNoException) {
+            assertThat(e.getMessage(), is("terminate authc process"));
+            assertThat(e.getHeader("WWW-Authenticate"), contains(basicScheme));
+        } else {
+            if (throwElasticsearchSecurityException) {
+                assertThat(e.getMessage(), is("authentication error"));
+                if (withAuthenticateHeader) {
+                    assertThat(e.getHeader("WWW-Authenticate"), contains(selectedScheme));
+                } else {
+                    assertThat(e.getHeader("WWW-Authenticate"), contains(basicScheme));
+                }
+            } else {
+                assertThat(e.getMessage(), is("error attempting to authenticate request"));
+                assertThat(e.getHeader("WWW-Authenticate"), contains(basicScheme));
+            }
+        }
+        verify(auditTrail).authenticationFailed(secondRealm.name(), token, "_action", message);
+        verify(auditTrail).authenticationFailed(token, "_action", message);
+        verifyNoMoreInteractions(auditTrail);
+    }
+
     public void testRealmAuthenticateThrowingException() throws Exception {
         AuthenticationToken token = mock(AuthenticationToken.class);
         when(secondRealm.token(threadContext)).thenReturn(token);
@@ -998,6 +1041,19 @@ public class AuthenticationServiceTests extends ESTestCase {
         }).when(realm).authenticate(eq(token), any(ActionListener.class));
     }
 
+    @SuppressWarnings("unchecked")
+    private void mockAuthenticate(Realm realm, AuthenticationToken token, Exception e, boolean terminate) {
+        doAnswer((i) -> {
+            ActionListener<AuthenticationResult> listener = (ActionListener<AuthenticationResult>) i.getArguments()[1];
+            if (terminate) {
+                listener.onResponse(AuthenticationResult.terminate("terminate authc process", e));
+            } else {
+                listener.onResponse(AuthenticationResult.unsuccessful("unsuccessful, but continue authc process", e));
+            }
+            return null;
+        }).when(realm).authenticate(eq(token), any(ActionListener.class));
+    }
+
     private Authentication authenticateBlocking(RestRequest restRequest) {
         PlainActionFuture<Authentication> future = new PlainActionFuture<>();
         service.authenticate(restRequest, future);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java
index 0cbeced00b2..2e91c40677e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java
@@ -14,6 +14,11 @@ import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
@@ -49,4 +54,12 @@ public class InternalRealmsTests extends ESTestCase {
             TestEnvironment.newEnvironment(settings), new ThreadContext(settings)));
         verify(securityIndex, times(2)).addIndexStateListener(isA(BiConsumer.class));
     }
+
+    public void testIsStandardType() {
+        String type = randomFrom(NativeRealmSettings.TYPE, FileRealmSettings.TYPE, LdapRealmSettings.AD_TYPE, LdapRealmSettings.LDAP_TYPE,
+                PkiRealmSettings.TYPE);
+        assertThat(InternalRealms.isStandardRealm(type), is(true));
+        type = randomFrom(SamlRealmSettings.TYPE, KerberosRealmSettings.TYPE);
+        assertThat(InternalRealms.isStandardRealm(type), is(false));
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
index ff4c30ddf8c..a71f5cb1cf7 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
@@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
 import org.elasticsearch.xpack.core.security.user.User;
@@ -335,10 +336,11 @@ public class RealmsTests extends ESTestCase {
     }
 
     public void testUnlicensedWithNonStandardRealms() throws Exception {
-        factories.put(SamlRealmSettings.TYPE, config -> new DummyRealm(SamlRealmSettings.TYPE, config));
+        final String selectedRealmType = randomFrom(SamlRealmSettings.TYPE, KerberosRealmSettings.TYPE);
+        factories.put(selectedRealmType, config -> new DummyRealm(selectedRealmType, config));
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir())
-                .put("xpack.security.authc.realms.foo.type", SamlRealmSettings.TYPE)
+                .put("xpack.security.authc.realms.foo.type", selectedRealmType)
                 .put("xpack.security.authc.realms.foo.order", "0");
         Settings settings = builder.build();
         Environment env = TestEnvironment.newEnvironment(settings);
@@ -349,7 +351,7 @@ public class RealmsTests extends ESTestCase {
         assertThat(realm, is(reservedRealm));
         assertThat(iter.hasNext(), is(true));
         realm = iter.next();
-        assertThat(realm.type(), is(SamlRealmSettings.TYPE));
+        assertThat(realm.type(), is(selectedRealmType));
         assertThat(iter.hasNext(), is(false));
 
         when(licenseState.allowedRealmType()).thenReturn(AllowedRealmType.DEFAULT);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationTokenTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationTokenTests.java
new file mode 100644
index 00000000000..eaba796b41f
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationTokenTests.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class KerberosAuthenticationTokenTests extends ESTestCase {
+
+    private static final String UNAUTHENTICATED_PRINCIPAL_NAME = "<Kerberos Token>";
+
+    public void testExtractTokenForValidAuthorizationHeader() throws IOException {
+        final String base64Token = Base64.getEncoder().encodeToString(randomAlphaOfLength(5).getBytes(StandardCharsets.UTF_8));
+        final String negotiate = randomBoolean() ? KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX : "negotiate ";
+        final String authzHeader = negotiate + base64Token;
+
+        final KerberosAuthenticationToken kerbAuthnToken = KerberosAuthenticationToken.extractToken(authzHeader);
+        assertNotNull(kerbAuthnToken);
+        assertEquals(UNAUTHENTICATED_PRINCIPAL_NAME, kerbAuthnToken.principal());
+        assertThat(kerbAuthnToken.credentials(), instanceOf((byte[].class)));
+        assertArrayEquals(Base64.getDecoder().decode(base64Token), (byte[]) kerbAuthnToken.credentials());
+    }
+
+    public void testExtractTokenForInvalidNegotiateAuthorizationHeaderShouldReturnNull() throws IOException {
+        final String header = randomFrom("negotiate", "Negotiate", " Negotiate", "NegotiateToken", "Basic ", " Custom ", null);
+        assertNull(KerberosAuthenticationToken.extractToken(header));
+    }
+
+    public void testExtractTokenForNegotiateAuthorizationHeaderWithNoTokenShouldThrowException() throws IOException {
+        final String header = randomFrom(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX, "negotiate ", "Negotiate      ");
+        final ElasticsearchSecurityException e =
+                expectThrows(ElasticsearchSecurityException.class, () -> KerberosAuthenticationToken.extractToken(header));
+        assertThat(e.getMessage(),
+                equalTo("invalid negotiate authentication header value, expected base64 encoded token but value is empty"));
+        assertContainsAuthenticateHeader(e);
+    }
+
+    public void testExtractTokenForNotBase64EncodedTokenThrowsException() throws IOException {
+        final String notBase64Token = "[B@6499375d";
+
+        final ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
+                () -> KerberosAuthenticationToken.extractToken(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX + notBase64Token));
+        assertThat(e.getMessage(),
+                equalTo("invalid negotiate authentication header value, could not decode base64 token " + notBase64Token));
+        assertContainsAuthenticateHeader(e);
+    }
+
+    public void testKerberoAuthenticationTokenClearCredentials() {
+        byte[] inputBytes = randomByteArrayOfLength(5);
+        final String base64Token = Base64.getEncoder().encodeToString(inputBytes);
+        final KerberosAuthenticationToken kerbAuthnToken =
+                KerberosAuthenticationToken.extractToken(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX + base64Token);
+        kerbAuthnToken.clearCredentials();
+        Arrays.fill(inputBytes, (byte) 0);
+        assertArrayEquals(inputBytes, (byte[]) kerbAuthnToken.credentials());
+    }
+
+    public void testEqualsHashCode() {
+        final KerberosAuthenticationToken kerberosAuthenticationToken =
+                new KerberosAuthenticationToken("base64EncodedToken".getBytes(StandardCharsets.UTF_8));
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(kerberosAuthenticationToken, (original) -> {
+            return new KerberosAuthenticationToken((byte[]) original.credentials());
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(kerberosAuthenticationToken, (original) -> {
+            byte[] originalCreds = (byte[]) original.credentials();
+            return new KerberosAuthenticationToken(Arrays.copyOf(originalCreds, originalCreds.length));
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(kerberosAuthenticationToken, (original) -> {
+            return new KerberosAuthenticationToken((byte[]) original.credentials());
+        }, KerberosAuthenticationTokenTests::mutateTestItem);
+    }
+
+    private static KerberosAuthenticationToken mutateTestItem(KerberosAuthenticationToken original) {
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                return new KerberosAuthenticationToken(randomByteArrayOfLength(10));
+            case 1:
+                return new KerberosAuthenticationToken("base64EncodedToken".getBytes(StandardCharsets.UTF_16));
+            case 2:
+                return new KerberosAuthenticationToken("[B@6499375d".getBytes(StandardCharsets.UTF_8));
+            default:
+                throw new IllegalArgumentException("unknown option");
+        }
+    }
+
+    private static void assertContainsAuthenticateHeader(ElasticsearchSecurityException e) {
+        assertThat(e.status(), is(RestStatus.UNAUTHORIZED));
+        assertThat(e.getHeaderKeys(), hasSize(1));
+        assertThat(e.getHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE), notNullValue());
+        assertThat(e.getHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE), contains(KerberosAuthenticationToken.NEGOTIATE_SCHEME_NAME));
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java
new file mode 100644
index 00000000000..5bc239241cf
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmAuthenticateFailedTests.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.ietf.jgss.GSSException;
+
+import java.nio.file.Path;
+import java.util.List;
+
+import javax.security.auth.login.LoginException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.AdditionalMatchers.aryEq;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.verify;
+
+public class KerberosRealmAuthenticateFailedTests extends KerberosRealmTestCase {
+
+    public void testAuthenticateWithNonKerberosAuthenticationToken() {
+        final KerberosRealm kerberosRealm = createKerberosRealm(randomAlphaOfLength(5));
+
+        final UsernamePasswordToken usernamePasswordToken =
+                new UsernamePasswordToken(randomAlphaOfLength(5), new SecureString(new char[] { 'a', 'b', 'c' }));
+        expectThrows(AssertionError.class, () -> kerberosRealm.authenticate(usernamePasswordToken, PlainActionFuture.newFuture()));
+    }
+
+    public void testAuthenticateDifferentFailureScenarios() throws LoginException, GSSException {
+        final String username = randomPrincipalName();
+        final String outToken = randomAlphaOfLength(10);
+        final KerberosRealm kerberosRealm = createKerberosRealm(username);
+        final boolean validTicket = rarely();
+        final boolean throwExceptionForInvalidTicket = validTicket ? false : randomBoolean();
+        final boolean throwLoginException = randomBoolean();
+        final byte[] decodedTicket = randomByteArrayOfLength(5);
+        final Path keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+        final boolean krbDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
+        if (validTicket) {
+            mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, outToken), null);
+        } else {
+            if (throwExceptionForInvalidTicket) {
+                if (throwLoginException) {
+                    mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, null, new LoginException("Login Exception"));
+                } else {
+                    mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, null, new GSSException(GSSException.FAILURE));
+                }
+            } else {
+                mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(null, outToken), null);
+            }
+        }
+        final boolean nullKerberosAuthnToken = rarely();
+        final KerberosAuthenticationToken kerberosAuthenticationToken =
+                nullKerberosAuthnToken ? null : new KerberosAuthenticationToken(decodedTicket);
+        if (nullKerberosAuthnToken) {
+            expectThrows(AssertionError.class,
+                    () -> kerberosRealm.authenticate(kerberosAuthenticationToken, PlainActionFuture.newFuture()));
+        } else {
+            final PlainActionFuture<AuthenticationResult> future = new PlainActionFuture<>();
+            kerberosRealm.authenticate(kerberosAuthenticationToken, future);
+            AuthenticationResult result = future.actionGet();
+            assertThat(result, is(notNullValue()));
+            if (validTicket) {
+                final String expectedUsername = maybeRemoveRealmName(username);
+                final User expectedUser = new User(expectedUsername, roles.toArray(new String[roles.size()]), null, null, null, true);
+                assertSuccessAuthenticationResult(expectedUser, outToken, result);
+            } else {
+                assertThat(result.getStatus(), is(equalTo(AuthenticationResult.Status.TERMINATE)));
+                if (throwExceptionForInvalidTicket == false) {
+                    assertThat(result.getException(), is(instanceOf(ElasticsearchSecurityException.class)));
+                    final List<String> wwwAuthnHeader = ((ElasticsearchSecurityException) result.getException())
+                            .getHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE);
+                    assertThat(wwwAuthnHeader, is(notNullValue()));
+                    assertThat(wwwAuthnHeader.get(0), is(equalTo(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX + outToken)));
+                    assertThat(result.getMessage(), is(equalTo("failed to authenticate user, gss context negotiation not complete")));
+                } else {
+                    if (throwLoginException) {
+                        assertThat(result.getMessage(), is(equalTo("failed to authenticate user, service login failure")));
+                    } else {
+                        assertThat(result.getMessage(), is(equalTo("failed to authenticate user, gss context negotiation failure")));
+                    }
+                    assertThat(result.getException(), is(instanceOf(ElasticsearchSecurityException.class)));
+                    final List<String> wwwAuthnHeader = ((ElasticsearchSecurityException) result.getException())
+                            .getHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE);
+                    assertThat(wwwAuthnHeader, is(notNullValue()));
+                    assertThat(wwwAuthnHeader.get(0), is(equalTo(KerberosAuthenticationToken.NEGOTIATE_SCHEME_NAME)));
+                }
+            }
+            verify(mockKerberosTicketValidator).validateTicket(aryEq(decodedTicket), eq(keytabPath), eq(krbDebug),
+                    any(ActionListener.class));
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java
new file mode 100644
index 00000000000..b6e1df9ddbb
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.RealmSettings;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class KerberosRealmBootstrapCheckTests extends ESTestCase {
+
+    public void testBootstrapCheckFailsForMultipleKerberosRealms() throws IOException {
+        final Path tempDir = createTempDir();
+        final Settings settings1 = buildKerberosRealmSettings("kerb1", false, tempDir);
+        final Settings settings2 = buildKerberosRealmSettings("kerb2", false, tempDir);
+        final Settings settings3 = realm("pki1", PkiRealmSettings.TYPE, Settings.builder()).build();
+        final Settings settings =
+                Settings.builder().put("path.home", tempDir).put(settings1).put(settings2).put(settings3).build();
+        final BootstrapContext context = new BootstrapContext(settings, null);
+        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
+                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
+        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
+        assertThat(result, is(notNullValue()));
+        assertThat(result.isFailure(), is(true));
+        assertThat(result.getMessage(), equalTo("multiple [" + KerberosRealmSettings.TYPE + "] realms are configured. ["
+                + KerberosRealmSettings.TYPE + "] can only have one such realm configured"));
+    }
+
+    public void testBootstrapCheckFailsForMissingKeytabFile() throws IOException {
+        final Path tempDir = createTempDir();
+        final Settings settings =
+                Settings.builder().put("path.home", tempDir).put(buildKerberosRealmSettings("kerb1", true, tempDir)).build();
+        final BootstrapContext context = new BootstrapContext(settings, null);
+        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
+                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
+        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
+        assertThat(result, is(notNullValue()));
+        assertThat(result.isFailure(), is(true));
+        assertThat(result.getMessage(),
+                equalTo("configured service key tab file [" + tempDir.resolve("kerb1.keytab").toString() + "] does not exist"));
+    }
+
+    public void testBootstrapCheckFailsForMissingRealmType() throws IOException {
+        final Path tempDir = createTempDir();
+        final String name = "kerb1";
+        final Settings settings1 = buildKerberosRealmSettings("kerb1", false, tempDir);
+        final Settings settings2 = realm(name, randomFrom("", "    "), Settings.builder()).build();
+        final Settings settings =
+                Settings.builder().put("path.home", tempDir).put(settings1).put(settings2).build();
+        final BootstrapContext context = new BootstrapContext(settings, null);
+        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
+                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
+        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
+        assertThat(result, is(notNullValue()));
+        assertThat(result.isFailure(), is(true));
+        assertThat(result.getMessage(), equalTo("missing realm type for [" + name + "] realm"));
+    }
+
+    public void testBootstrapCheckSucceedsForCorrectConfiguration() throws IOException {
+        final Path tempDir = createTempDir();
+        final Settings finalSettings =
+                Settings.builder().put("path.home", tempDir).put(buildKerberosRealmSettings("kerb1", false, tempDir)).build();
+        final BootstrapContext context = new BootstrapContext(finalSettings, null);
+        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
+                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(finalSettings));
+        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
+        assertThat(result, is(notNullValue()));
+        assertThat(result.isSuccess(), is(true));
+    }
+
+    public void testBootstrapCheckSucceedsForNoKerberosRealms() throws IOException {
+        final Path tempDir = createTempDir();
+        final Settings finalSettings = Settings.builder().put("path.home", tempDir).build();
+        final BootstrapContext context = new BootstrapContext(finalSettings, null);
+        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
+                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(finalSettings));
+        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
+        assertThat(result, is(notNullValue()));
+        assertThat(result.isSuccess(), is(true));
+    }
+
+    private Settings buildKerberosRealmSettings(final String name, final boolean missingKeytab, final Path tempDir) throws IOException {
+        final Settings.Builder builder = Settings.builder();
+        if (missingKeytab == false) {
+            KerberosTestCase.writeKeyTab(tempDir.resolve(name + ".keytab"), null);
+        }
+        builder.put(KerberosTestCase.buildKerberosRealmSettings(tempDir.resolve(name + ".keytab").toString()));
+        return realm(name, KerberosRealmSettings.TYPE, builder).build();
+    }
+
+    private Settings.Builder realm(final String name, final String type, final Settings.Builder settings) {
+        final String prefix = RealmSettings.PREFIX + name + ".";
+        if (type != null) {
+            settings.put("type", type);
+        }
+        final Settings.Builder builder = Settings.builder().put(settings.normalizePrefix(prefix).build(), false);
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java
new file mode 100644
index 00000000000..69ebe15c5d7
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData;
+import org.ietf.jgss.GSSException;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.security.auth.login.LoginException;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.AdditionalMatchers.aryEq;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class KerberosRealmCacheTests extends KerberosRealmTestCase {
+
+    public void testAuthenticateWithCache() throws LoginException, GSSException {
+        final String username = randomPrincipalName();
+        final String outToken = randomAlphaOfLength(10);
+        final KerberosRealm kerberosRealm = createKerberosRealm(username);
+
+        final String expectedUsername = maybeRemoveRealmName(username);
+        final User expectedUser = new User(expectedUsername, roles.toArray(new String[roles.size()]), null, null, null, true);
+        final byte[] decodedTicket = randomByteArrayOfLength(10);
+        final Path keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+        final boolean krbDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
+        mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, outToken), null);
+        final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket);
+
+        // authenticate
+        final User user1 = authenticateAndAssertResult(kerberosRealm, expectedUser, kerberosAuthenticationToken, outToken);
+        // authenticate with cache
+        final User user2 = authenticateAndAssertResult(kerberosRealm, expectedUser, kerberosAuthenticationToken, outToken);
+
+        assertThat(user1, sameInstance(user2));
+        verify(mockKerberosTicketValidator, times(2)).validateTicket(aryEq(decodedTicket), eq(keytabPath), eq(krbDebug),
+                any(ActionListener.class));
+        verify(mockNativeRoleMappingStore).refreshRealmOnChange(kerberosRealm);
+        verify(mockNativeRoleMappingStore).resolveRoles(any(UserData.class), any(ActionListener.class));
+        verifyNoMoreInteractions(mockKerberosTicketValidator, mockNativeRoleMappingStore);
+    }
+
+    public void testCacheInvalidationScenarios() throws LoginException, GSSException {
+        final String outToken = randomAlphaOfLength(10);
+        final List<String> userNames = Arrays.asList(randomPrincipalName(), randomPrincipalName());
+        final KerberosRealm kerberosRealm = createKerberosRealm(userNames.toArray(new String[0]));
+        verify(mockNativeRoleMappingStore).refreshRealmOnChange(kerberosRealm);
+
+        final String authNUsername = randomFrom(userNames);
+        final byte[] decodedTicket = randomByteArrayOfLength(10);
+        final Path keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+        final boolean krbDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
+        mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(authNUsername, outToken), null);
+        final String expectedUsername = maybeRemoveRealmName(authNUsername);
+        final User expectedUser = new User(expectedUsername, roles.toArray(new String[roles.size()]), null, null, null, true);
+
+        final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket);
+        final User user1 = authenticateAndAssertResult(kerberosRealm, expectedUser, kerberosAuthenticationToken, outToken);
+
+        final String expireThisUser = randomFrom(userNames);
+        boolean expireAll = randomBoolean();
+        if (expireAll) {
+            kerberosRealm.expireAll();
+        } else {
+            kerberosRealm.expire(maybeRemoveRealmName(expireThisUser));
+        }
+
+        final User user2 = authenticateAndAssertResult(kerberosRealm, expectedUser, kerberosAuthenticationToken, outToken);
+
+        if (expireAll || expireThisUser.equals(authNUsername)) {
+            assertThat(user1, is(not(sameInstance(user2))));
+            verify(mockNativeRoleMappingStore, times(2)).resolveRoles(any(UserData.class), any(ActionListener.class));
+        } else {
+            assertThat(user1, sameInstance(user2));
+            verify(mockNativeRoleMappingStore).resolveRoles(any(UserData.class), any(ActionListener.class));
+        }
+        verify(mockKerberosTicketValidator, times(2)).validateTicket(aryEq(decodedTicket), eq(keytabPath), eq(krbDebug),
+                any(ActionListener.class));
+        verifyNoMoreInteractions(mockKerberosTicketValidator, mockNativeRoleMappingStore);
+    }
+
+    public void testAuthenticateWithValidTicketSucessAuthnWithUserDetailsWhenCacheDisabled()
+            throws LoginException, GSSException, IOException {
+        // if cache.ttl <= 0 then the cache is disabled
+        settings = KerberosTestCase.buildKerberosRealmSettings(
+                KerberosTestCase.writeKeyTab(dir.resolve("key.keytab"), randomAlphaOfLength(4)).toString(), 100, "0m", true,
+                randomBoolean());
+        final String username = randomPrincipalName();
+        final String outToken = randomAlphaOfLength(10);
+        final KerberosRealm kerberosRealm = createKerberosRealm(username);
+
+        final String expectedUsername = maybeRemoveRealmName(username);
+        final User expectedUser = new User(expectedUsername, roles.toArray(new String[roles.size()]), null, null, null, true);
+        final byte[] decodedTicket = randomByteArrayOfLength(10);
+        final Path keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+        final boolean krbDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
+        mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, outToken), null);
+        final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket);
+
+        // authenticate
+        final User user1 = authenticateAndAssertResult(kerberosRealm, expectedUser, kerberosAuthenticationToken, outToken);
+        // authenticate when cache has been disabled
+        final User user2 = authenticateAndAssertResult(kerberosRealm, expectedUser, kerberosAuthenticationToken, outToken);
+
+        assertThat(user1, not(sameInstance(user2)));
+        verify(mockKerberosTicketValidator, times(2)).validateTicket(aryEq(decodedTicket), eq(keytabPath), eq(krbDebug),
+                any(ActionListener.class));
+        verify(mockNativeRoleMappingStore).refreshRealmOnChange(kerberosRealm);
+        verify(mockNativeRoleMappingStore, times(2)).resolveRoles(any(UserData.class), any(ActionListener.class));
+        verifyNoMoreInteractions(mockKerberosTicketValidator, mockNativeRoleMappingStore);
+    }
+
+    private User authenticateAndAssertResult(final KerberosRealm kerberosRealm, final User expectedUser,
+            final KerberosAuthenticationToken kerberosAuthenticationToken, String outToken) {
+        final PlainActionFuture<AuthenticationResult> future = PlainActionFuture.newFuture();
+        kerberosRealm.authenticate(kerberosAuthenticationToken, future);
+        final AuthenticationResult result = future.actionGet();
+        assertSuccessAuthenticationResult(expectedUser, outToken, result);
+        return result.getUser();
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmSettingsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmSettingsTests.java
new file mode 100644
index 00000000000..2e47d03d49d
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmSettingsTests.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class KerberosRealmSettingsTests extends ESTestCase {
+
+    public void testKerberosRealmSettings() throws IOException {
+        final Path dir = createTempDir();
+        Path configDir = dir.resolve("config");
+        if (Files.exists(configDir) == false) {
+            configDir = Files.createDirectory(configDir);
+        }
+        final String keytabPathConfig = "config" + dir.getFileSystem().getSeparator() + "http.keytab";
+        KerberosTestCase.writeKeyTab(dir.resolve(keytabPathConfig), null);
+        final Integer maxUsers = randomInt();
+        final String cacheTTL = randomLongBetween(10L, 100L) + "m";
+        final boolean enableDebugLogs = randomBoolean();
+        final boolean removeRealmName = randomBoolean();
+        final Settings settings = KerberosTestCase.buildKerberosRealmSettings(keytabPathConfig, maxUsers, cacheTTL, enableDebugLogs,
+                removeRealmName);
+
+        assertThat(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(settings), equalTo(keytabPathConfig));
+        assertThat(KerberosRealmSettings.CACHE_TTL_SETTING.get(settings),
+                equalTo(TimeValue.parseTimeValue(cacheTTL, KerberosRealmSettings.CACHE_TTL_SETTING.getKey())));
+        assertThat(KerberosRealmSettings.CACHE_MAX_USERS_SETTING.get(settings), equalTo(maxUsers));
+        assertThat(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(settings), is(enableDebugLogs));
+        assertThat(KerberosRealmSettings.SETTING_REMOVE_REALM_NAME.get(settings), is(removeRealmName));
+    }
+
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java
new file mode 100644
index 00000000000..9c2c6484c82
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
+import org.elasticsearch.xpack.security.support.SecurityIndexManager;
+import org.junit.After;
+import org.junit.Before;
+
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.AdditionalMatchers.aryEq;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+public abstract class KerberosRealmTestCase extends ESTestCase {
+
+    protected Path dir;
+    protected ThreadPool threadPool;
+    protected Settings globalSettings;
+    protected ResourceWatcherService resourceWatcherService;
+    protected Settings settings;
+    protected RealmConfig config;
+
+    protected KerberosTicketValidator mockKerberosTicketValidator;
+    protected NativeRoleMappingStore mockNativeRoleMappingStore;
+
+    protected static final Set<String> roles = Sets.newHashSet("admin", "kibana_user");
+
+    @Before
+    public void setup() throws Exception {
+        threadPool = new TestThreadPool("kerb realm tests");
+        resourceWatcherService = new ResourceWatcherService(Settings.EMPTY, threadPool);
+        dir = createTempDir();
+        globalSettings = Settings.builder().put("path.home", dir).build();
+        settings = KerberosTestCase.buildKerberosRealmSettings(KerberosTestCase.writeKeyTab(dir.resolve("key.keytab"), "asa").toString(),
+                100, "10m", true, randomBoolean());
+    }
+
+    @After
+    public void shutdown() throws InterruptedException {
+        resourceWatcherService.stop();
+        terminate(threadPool);
+    }
+
+    protected void mockKerberosTicketValidator(final byte[] decodedTicket, final Path keytabPath, final boolean krbDebug,
+            final Tuple<String, String> value, final Exception e) {
+        assert value != null || e != null;
+        doAnswer((i) -> {
+            ActionListener<Tuple<String, String>> listener = (ActionListener<Tuple<String, String>>) i.getArguments()[3];
+            if (e != null) {
+                listener.onFailure(e);
+            } else {
+                listener.onResponse(value);
+            }
+            return null;
+        }).when(mockKerberosTicketValidator).validateTicket(aryEq(decodedTicket), eq(keytabPath), eq(krbDebug), any(ActionListener.class));
+    }
+
+    protected void assertSuccessAuthenticationResult(final User expectedUser, final String outToken, final AuthenticationResult result) {
+        assertThat(result, is(notNullValue()));
+        assertThat(result.getStatus(), is(equalTo(AuthenticationResult.Status.SUCCESS)));
+        assertThat(result.getUser(), is(equalTo(expectedUser)));
+        final Map<String, List<String>> responseHeaders = threadPool.getThreadContext().getResponseHeaders();
+        assertThat(responseHeaders, is(notNullValue()));
+        assertThat(responseHeaders.get(KerberosAuthenticationToken.WWW_AUTHENTICATE).get(0),
+                is(equalTo(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX + outToken)));
+    }
+
+    protected KerberosRealm createKerberosRealm(final String... userForRoleMapping) {
+        config = new RealmConfig("test-kerb-realm", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
+                new ThreadContext(globalSettings));
+        mockNativeRoleMappingStore = roleMappingStore(Arrays.asList(userForRoleMapping));
+        mockKerberosTicketValidator = mock(KerberosTicketValidator.class);
+        final KerberosRealm kerberosRealm =
+                new KerberosRealm(config, mockNativeRoleMappingStore, mockKerberosTicketValidator, threadPool, null);
+        return kerberosRealm;
+    }
+
+    @SuppressWarnings("unchecked")
+    protected NativeRoleMappingStore roleMappingStore(final List<String> userNames) {
+        final List<String> expectedUserNames = userNames.stream().map(this::maybeRemoveRealmName).collect(Collectors.toList());
+        final Client mockClient = mock(Client.class);
+        when(mockClient.threadPool()).thenReturn(threadPool);
+        when(mockClient.settings()).thenReturn(settings);
+
+        final NativeRoleMappingStore store = new NativeRoleMappingStore(Settings.EMPTY, mockClient, mock(SecurityIndexManager.class));
+        final NativeRoleMappingStore roleMapper = spy(store);
+
+        doAnswer(invocation -> {
+            final UserRoleMapper.UserData userData = (UserRoleMapper.UserData) invocation.getArguments()[0];
+            final ActionListener<Set<String>> listener = (ActionListener<Set<String>>) invocation.getArguments()[1];
+            if (expectedUserNames.contains(userData.getUsername())) {
+                listener.onResponse(roles);
+            } else {
+                listener.onFailure(
+                        Exceptions.authorizationError("Expected UPN '" + expectedUserNames + "' but was '" + userData.getUsername() + "'"));
+            }
+            return null;
+        }).when(roleMapper).resolveRoles(any(UserRoleMapper.UserData.class), any(ActionListener.class));
+
+        return roleMapper;
+    }
+
+    protected String randomPrincipalName() {
+        final StringBuilder principalName = new StringBuilder();
+        principalName.append(randomAlphaOfLength(5));
+        final boolean withInstance = randomBoolean();
+        if (withInstance) {
+            principalName.append("/").append(randomAlphaOfLength(5));
+        }
+        principalName.append(randomAlphaOfLength(5).toUpperCase(Locale.ROOT));
+        return principalName.toString();
+    }
+
+    /**
+     * Usually principal names are in the form 'user/instance@REALM'. This method
+     * removes '@REALM' part from the principal name if
+     * {@link KerberosRealmSettings#SETTING_REMOVE_REALM_NAME} is {@code true} else
+     * will return the input string.
+     *
+     * @param principalName user principal name
+     * @return username after removal of realm
+     */
+    protected String maybeRemoveRealmName(final String principalName) {
+        if (KerberosRealmSettings.SETTING_REMOVE_REALM_NAME.get(settings)) {
+            int foundAtIndex = principalName.indexOf('@');
+            if (foundAtIndex > 0) {
+                return principalName.substring(0, foundAtIndex);
+            }
+        }
+        return principalName;
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
new file mode 100644
index 00000000000..43536abaf29
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData;
+import org.ietf.jgss.GSSException;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import javax.security.auth.login.LoginException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.AdditionalMatchers.aryEq;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class KerberosRealmTests extends KerberosRealmTestCase {
+
+    public void testSupports() {
+        final KerberosRealm kerberosRealm = createKerberosRealm("test@REALM");
+
+        final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(randomByteArrayOfLength(5));
+        assertThat(kerberosRealm.supports(kerberosAuthenticationToken), is(true));
+        final UsernamePasswordToken usernamePasswordToken =
+                new UsernamePasswordToken(randomAlphaOfLength(5), new SecureString(new char[] { 'a', 'b', 'c' }));
+        assertThat(kerberosRealm.supports(usernamePasswordToken), is(false));
+    }
+
+    public void testAuthenticateWithValidTicketSucessAuthnWithUserDetails() throws LoginException, GSSException {
+        final String username = randomPrincipalName();
+        final KerberosRealm kerberosRealm = createKerberosRealm(username);
+        final String expectedUsername = maybeRemoveRealmName(username);
+        final User expectedUser = new User(expectedUsername, roles.toArray(new String[roles.size()]), null, null, null, true);
+        final byte[] decodedTicket = "base64encodedticket".getBytes(StandardCharsets.UTF_8);
+        final Path keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+        final boolean krbDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
+        mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>(username, "out-token"), null);
+        final KerberosAuthenticationToken kerberosAuthenticationToken = new KerberosAuthenticationToken(decodedTicket);
+
+        final PlainActionFuture<AuthenticationResult> future = new PlainActionFuture<>();
+        kerberosRealm.authenticate(kerberosAuthenticationToken, future);
+        assertSuccessAuthenticationResult(expectedUser, "out-token", future.actionGet());
+
+        verify(mockKerberosTicketValidator, times(1)).validateTicket(aryEq(decodedTicket), eq(keytabPath), eq(krbDebug),
+                any(ActionListener.class));
+        verify(mockNativeRoleMappingStore).refreshRealmOnChange(kerberosRealm);
+        verify(mockNativeRoleMappingStore).resolveRoles(any(UserData.class), any(ActionListener.class));
+        verifyNoMoreInteractions(mockKerberosTicketValidator, mockNativeRoleMappingStore);
+    }
+
+    public void testFailedAuthorization() throws LoginException, GSSException {
+        final String username = randomPrincipalName();
+        final KerberosRealm kerberosRealm = createKerberosRealm(username);
+        final byte[] decodedTicket = "base64encodedticket".getBytes(StandardCharsets.UTF_8);
+        final Path keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+        final boolean krbDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
+        mockKerberosTicketValidator(decodedTicket, keytabPath, krbDebug, new Tuple<>("does-not-exist@REALM", "out-token"), null);
+
+        final PlainActionFuture<AuthenticationResult> future = new PlainActionFuture<>();
+        kerberosRealm.authenticate(new KerberosAuthenticationToken(decodedTicket), future);
+
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
+        assertThat(e.status(), is(RestStatus.FORBIDDEN));
+        assertThat(e.getMessage(), equalTo("Expected UPN '" + Arrays.asList(maybeRemoveRealmName(username)) + "' but was '"
+                + maybeRemoveRealmName("does-not-exist@REALM") + "'"));
+    }
+
+    public void testLookupUser() {
+        final String username = randomPrincipalName();
+        final KerberosRealm kerberosRealm = createKerberosRealm(username);
+        final PlainActionFuture<User> future = new PlainActionFuture<>();
+        kerberosRealm.lookupUser(username, future);
+        assertThat(future.actionGet(), is(nullValue()));
+    }
+
+}
\ No newline at end of file
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
new file mode 100644
index 00000000000..891f400c7be
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import javax.security.auth.Subject;
+
+/**
+ * Base Test class for Kerberos.
+ * <p>
+ * Takes care of starting {@link SimpleKdcLdapServer} as Kdc server backed by
+ * Ldap Server.
+ * <p>
+ * Also assists in building principal names, creation of principals and realm
+ * settings.
+ */
+public abstract class KerberosTestCase extends ESTestCase {
+
+    protected Settings globalSettings;
+    protected Settings settings;
+    protected List<String> serviceUserNames;
+    protected List<String> clientUserNames;
+    protected Path workDir = null;
+
+    protected SimpleKdcLdapServer simpleKdcLdapServer;
+
+    private static Locale restoreLocale;
+    private static Set<String> unsupportedLocaleLanguages;
+    static {
+        unsupportedLocaleLanguages = new HashSet<>();
+        /*
+         * arabic and other languages have problem due to handling of GeneralizedTime in
+         * SimpleKdcServer For more look at :
+         * org.apache.kerby.asn1.type.Asn1GeneralizedTime#toBytes()
+         */
+        unsupportedLocaleLanguages.add("ar");
+        unsupportedLocaleLanguages.add("ja");
+        unsupportedLocaleLanguages.add("th");
+        unsupportedLocaleLanguages.add("hi");
+        unsupportedLocaleLanguages.add("uz");
+        unsupportedLocaleLanguages.add("fa");
+        unsupportedLocaleLanguages.add("ks");
+    }
+
+    @BeforeClass
+    public static void setupKerberos() throws Exception {
+        if (isLocaleUnsupported()) {
+            Logger logger = Loggers.getLogger(KerberosTestCase.class);
+            logger.warn("Attempting to run Kerberos test on {} locale, but that breaks SimpleKdcServer. Switching to English.",
+                    Locale.getDefault());
+            restoreLocale = Locale.getDefault();
+            Locale.setDefault(Locale.ENGLISH);
+        }
+    }
+
+    @AfterClass
+    public static void restoreLocale() throws Exception {
+        if (restoreLocale != null) {
+            Locale.setDefault(restoreLocale);
+            restoreLocale = null;
+        }
+    }
+
+    private static boolean isLocaleUnsupported() {
+        return unsupportedLocaleLanguages.contains(Locale.getDefault().getLanguage());
+    }
+
+    @Before
+    public void startSimpleKdcLdapServer() throws Exception {
+        workDir = createTempDir();
+        globalSettings = Settings.builder().put("path.home", workDir).build();
+
+        final Path kdcLdiff = getDataPath("/kdc.ldiff");
+        simpleKdcLdapServer = new SimpleKdcLdapServer(workDir, "com", "example", kdcLdiff);
+
+        // Create SPNs and UPNs
+        serviceUserNames = new ArrayList<>();
+        Randomness.get().ints(randomIntBetween(1, 6)).forEach((i) -> {
+            serviceUserNames.add("HTTP/" + randomAlphaOfLength(8));
+        });
+        final Path ktabPathForService = createPrincipalKeyTab(workDir, serviceUserNames.toArray(new String[0]));
+        clientUserNames = new ArrayList<>();
+        Randomness.get().ints(randomIntBetween(1, 6)).forEach((i) -> {
+            String clientUserName = "client-" + randomAlphaOfLength(8);
+            clientUserNames.add(clientUserName);
+            try {
+                createPrincipal(clientUserName, "pwd".toCharArray());
+            } catch (Exception e) {
+                throw ExceptionsHelper.convertToRuntime(e);
+            }
+        });
+        settings = buildKerberosRealmSettings(ktabPathForService.toString());
+    }
+
+    @After
+    public void tearDownMiniKdc() throws IOException, PrivilegedActionException {
+        simpleKdcLdapServer.stop();
+    }
+
+    /**
+     * Creates principals and exports them to the keytab created in the directory.
+     *
+     * @param dir Directory where the key tab would be created.
+     * @param princNames principal names to be created
+     * @return {@link Path} to key tab file.
+     * @throws Exception
+     */
+    protected Path createPrincipalKeyTab(final Path dir, final String... princNames) throws Exception {
+        final Path path = dir.resolve(randomAlphaOfLength(10) + ".keytab");
+        simpleKdcLdapServer.createPrincipal(path, princNames);
+        return path;
+    }
+
+    /**
+     * Creates principal with given name and password.
+     * 
+     * @param principalName Principal name
+     * @param password Password
+     * @throws Exception
+     */
+    protected void createPrincipal(final String principalName, final char[] password) throws Exception {
+        simpleKdcLdapServer.createPrincipal(principalName, new String(password));
+    }
+
+    /**
+     * Appends realm name to user to form principal name
+     *
+     * @param user user name
+     * @return principal name in the form user@REALM
+     */
+    protected String principalName(final String user) {
+        return user + "@" + simpleKdcLdapServer.getRealm();
+    }
+
+    /**
+     * Invokes Subject.doAs inside a doPrivileged block
+     *
+     * @param subject {@link Subject}
+     * @param action {@link PrivilegedExceptionAction} action for performing inside
+     *            Subject.doAs
+     * @return <T> Type of value as returned by PrivilegedAction
+     * @throws PrivilegedActionException
+     */
+    static <T> T doAsWrapper(final Subject subject, final PrivilegedExceptionAction<T> action) throws PrivilegedActionException {
+        return AccessController.doPrivileged((PrivilegedExceptionAction<T>) () -> Subject.doAs(subject, action));
+    }
+
+    /**
+     * Write content to provided keytab file.
+     * 
+     * @param keytabPath {@link Path} to keytab file.
+     * @param content Content for keytab
+     * @return key tab path
+     * @throws IOException
+     */
+    public static Path writeKeyTab(final Path keytabPath, final String content) throws IOException {
+        try (BufferedWriter bufferedWriter = Files.newBufferedWriter(keytabPath, StandardCharsets.US_ASCII)) {
+            bufferedWriter.write(Strings.isNullOrEmpty(content) ? "test-content" : content);
+        }
+        return keytabPath;
+    }
+
+    /**
+     * Build kerberos realm settings with default config and given keytab
+     * 
+     * @param keytabPath key tab file path
+     * @return {@link Settings} for kerberos realm
+     */
+    public static Settings buildKerberosRealmSettings(final String keytabPath) {
+        return buildKerberosRealmSettings(keytabPath, 100, "10m", true, false);
+    }
+
+    /**
+     * Build kerberos realm settings
+     * 
+     * @param keytabPath key tab file path
+     * @param maxUsersInCache max users to be maintained in cache
+     * @param cacheTTL time to live for cached entries
+     * @param enableDebugging for krb5 logs
+     * @param removeRealmName {@code true} if we want to remove realm name from the username of form 'user@REALM'
+     * @return {@link Settings} for kerberos realm
+     */
+    public static Settings buildKerberosRealmSettings(final String keytabPath, final int maxUsersInCache, final String cacheTTL,
+            final boolean enableDebugging, final boolean removeRealmName) {
+        final Settings.Builder builder = Settings.builder().put(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.getKey(), keytabPath)
+                .put(KerberosRealmSettings.CACHE_MAX_USERS_SETTING.getKey(), maxUsersInCache)
+                .put(KerberosRealmSettings.CACHE_TTL_SETTING.getKey(), cacheTTL)
+                .put(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.getKey(), enableDebugging)
+                .put(KerberosRealmSettings.SETTING_REMOVE_REALM_NAME.getKey(), removeRealmName);
+        return builder.build();
+    }
+
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidatorTests.java
new file mode 100644
index 00000000000..8f35e0bde44
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidatorTests.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.ietf.jgss.GSSException;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.security.PrivilegedActionException;
+import java.util.Base64;
+import java.util.concurrent.ExecutionException;
+
+import javax.security.auth.login.LoginException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class KerberosTicketValidatorTests extends KerberosTestCase {
+
+    private KerberosTicketValidator kerberosTicketValidator = new KerberosTicketValidator();
+
+    public void testKerbTicketGeneratedForDifferentServerFailsValidation() throws Exception {
+        createPrincipalKeyTab(workDir, "differentServer");
+
+        // Client login and init token preparation
+        final String clientUserName = randomFrom(clientUserNames);
+        try (SpnegoClient spnegoClient =
+                new SpnegoClient(principalName(clientUserName), new SecureString("pwd".toCharArray()), principalName("differentServer"));) {
+            final String base64KerbToken = spnegoClient.getBase64EncodedTokenForSpnegoHeader();
+            assertThat(base64KerbToken, is(notNullValue()));
+
+            final Environment env = TestEnvironment.newEnvironment(globalSettings);
+            final Path keytabPath = env.configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(settings));
+            final PlainActionFuture<Tuple<String, String>> future = new PlainActionFuture<>();
+            kerberosTicketValidator.validateTicket(Base64.getDecoder().decode(base64KerbToken), keytabPath, true, future);
+            final GSSException gssException = expectThrows(GSSException.class, () -> unwrapExpectedExceptionFromFutureAndThrow(future));
+            assertThat(gssException.getMajor(), equalTo(GSSException.FAILURE));
+        }
+    }
+
+    public void testInvalidKerbTicketFailsValidation() throws Exception {
+        final String base64KerbToken = Base64.getEncoder().encodeToString(randomByteArrayOfLength(5));
+
+        final Environment env = TestEnvironment.newEnvironment(globalSettings);
+        final Path keytabPath = env.configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(settings));
+        kerberosTicketValidator.validateTicket(Base64.getDecoder().decode(base64KerbToken), keytabPath, true,
+                new ActionListener<Tuple<String, String>>() {
+                    boolean exceptionHandled = false;
+
+                    @Override
+                    public void onResponse(Tuple<String, String> response) {
+                        fail("expected exception to be thrown of type GSSException");
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        assertThat(exceptionHandled, is(false));
+                        assertThat(e, instanceOf(GSSException.class));
+                        assertThat(((GSSException) e).getMajor(), equalTo(GSSException.DEFECTIVE_TOKEN));
+                        exceptionHandled = true;
+                    }
+                });
+    }
+
+    public void testWhenKeyTabWithInvalidContentFailsValidation()
+            throws LoginException, GSSException, IOException, PrivilegedActionException {
+        // Client login and init token preparation
+        final String clientUserName = randomFrom(clientUserNames);
+        try (SpnegoClient spnegoClient = new SpnegoClient(principalName(clientUserName), new SecureString("pwd".toCharArray()),
+                principalName(randomFrom(serviceUserNames)));) {
+            final String base64KerbToken = spnegoClient.getBase64EncodedTokenForSpnegoHeader();
+            assertThat(base64KerbToken, is(notNullValue()));
+
+            final Path ktabPath = writeKeyTab(workDir.resolve("invalid.keytab"), "not - a - valid - key - tab");
+            settings = buildKerberosRealmSettings(ktabPath.toString());
+            final Environment env = TestEnvironment.newEnvironment(globalSettings);
+            final Path keytabPath = env.configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(settings));
+            final PlainActionFuture<Tuple<String, String>> future = new PlainActionFuture<>();
+            kerberosTicketValidator.validateTicket(Base64.getDecoder().decode(base64KerbToken), keytabPath, true, future);
+            final GSSException gssException = expectThrows(GSSException.class, () -> unwrapExpectedExceptionFromFutureAndThrow(future));
+            assertThat(gssException.getMajor(), equalTo(GSSException.FAILURE));
+        }
+    }
+
+    public void testValidKebrerosTicket() throws PrivilegedActionException, GSSException, LoginException {
+        // Client login and init token preparation
+        final String clientUserName = randomFrom(clientUserNames);
+        try (SpnegoClient spnegoClient = new SpnegoClient(principalName(clientUserName), new SecureString("pwd".toCharArray()),
+                principalName(randomFrom(serviceUserNames)));) {
+            final String base64KerbToken = spnegoClient.getBase64EncodedTokenForSpnegoHeader();
+            assertThat(base64KerbToken, is(notNullValue()));
+
+            final Environment env = TestEnvironment.newEnvironment(globalSettings);
+            final Path keytabPath = env.configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(settings));
+            final PlainActionFuture<Tuple<String, String>> future = new PlainActionFuture<>();
+            kerberosTicketValidator.validateTicket(Base64.getDecoder().decode(base64KerbToken), keytabPath, true, future);
+            assertThat(future.actionGet(), is(notNullValue()));
+            assertThat(future.actionGet().v1(), equalTo(principalName(clientUserName)));
+            assertThat(future.actionGet().v2(), is(notNullValue()));
+
+            final String outToken = spnegoClient.handleResponse(future.actionGet().v2());
+            assertThat(outToken, is(nullValue()));
+            assertThat(spnegoClient.isEstablished(), is(true));
+        }
+    }
+
+    private void unwrapExpectedExceptionFromFutureAndThrow(PlainActionFuture<Tuple<String, String>> future) throws Throwable {
+        try {
+            future.actionGet();
+        } catch (Throwable t) {
+            Throwable throwThis = t;
+            while (throwThis instanceof UncategorizedExecutionException || throwThis instanceof ExecutionException) {
+                throwThis = throwThis.getCause();
+            }
+            throw throwThis;
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
new file mode 100644
index 00000000000..426cacb1a03
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
@@ -0,0 +1,224 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import com.unboundid.ldap.listener.InMemoryDirectoryServer;
+import com.unboundid.ldap.listener.InMemoryDirectoryServerConfig;
+
+import org.apache.kerby.kerberos.kerb.KrbException;
+import org.apache.kerby.kerberos.kerb.client.KrbConfig;
+import org.apache.kerby.kerberos.kerb.server.KdcConfigKey;
+import org.apache.kerby.kerberos.kerb.server.SimpleKdcServer;
+import org.apache.kerby.util.NetworkUtil;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Locale;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Utility wrapper around Apache {@link SimpleKdcServer} backed by Unboundid
+ * {@link InMemoryDirectoryServer}.<br>
+ * Starts in memory Ldap server and then uses it as backend for Kdc Server.
+ */
+public class SimpleKdcLdapServer {
+    private static final Logger logger = Loggers.getLogger(SimpleKdcLdapServer.class);
+
+    private Path workDir = null;
+    private SimpleKdcServer simpleKdc;
+    private InMemoryDirectoryServer ldapServer;
+
+    // KDC properties
+    private String transport = ESTestCase.randomFrom("TCP", "UDP");
+    private int kdcPort = 0;
+    private String host;
+    private String realm;
+    private boolean krb5DebugBackupConfigValue;
+
+    // LDAP properties
+    private String baseDn;
+    private Path ldiff;
+    private int ldapPort;
+
+    /**
+     * Constructor for SimpleKdcLdapServer, creates instance of Kdc server and ldap
+     * backend server. Also initializes and starts them with provided configuration.
+     * <p>
+     * To stop the KDC and ldap server use {@link #stop()}
+     *
+     * @param workDir Base directory for server, used to locate kdc.conf,
+     *            backend.conf and kdc.ldiff
+     * @param orgName Org name for base dn
+     * @param domainName domain name for base dn
+     * @param ldiff for ldap directory.
+     * @throws Exception
+     */
+    public SimpleKdcLdapServer(final Path workDir, final String orgName, final String domainName, final Path ldiff) throws Exception {
+        this.workDir = workDir;
+        this.realm = domainName.toUpperCase(Locale.ROOT) + "." + orgName.toUpperCase(Locale.ROOT);
+        this.baseDn = "dc=" + domainName + ",dc=" + orgName;
+        this.ldiff = ldiff;
+        this.krb5DebugBackupConfigValue = AccessController.doPrivileged(new PrivilegedExceptionAction<Boolean>() {
+            @Override
+            @SuppressForbidden(reason = "set or clear system property krb5 debug in kerberos tests")
+            public Boolean run() throws Exception {
+                boolean oldDebugSetting = Boolean.parseBoolean(System.getProperty("sun.security.krb5.debug"));
+                System.setProperty("sun.security.krb5.debug", Boolean.TRUE.toString());
+                return oldDebugSetting;
+            }
+        });
+
+        AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
+            @Override
+            public Void run() throws Exception {
+                init();
+                return null;
+            }
+        });
+        logger.info("SimpleKdcLdapServer started.");
+    }
+
+    @SuppressForbidden(reason = "Uses Apache Kdc which requires usage of java.io.File in order to create a SimpleKdcServer")
+    private void init() throws Exception {
+        // start ldap server
+        createLdapServiceAndStart();
+        // create ldap backend conf
+        createLdapBackendConf();
+        // Kdc Server
+        simpleKdc = new SimpleKdcServer(this.workDir.toFile(), new KrbConfig());
+        prepareKdcServerAndStart();
+    }
+
+    private void createLdapServiceAndStart() throws Exception {
+        InMemoryDirectoryServerConfig config = new InMemoryDirectoryServerConfig(baseDn);
+        config.setSchema(null);
+        ldapServer = new InMemoryDirectoryServer(config);
+        ldapServer.importFromLDIF(true, this.ldiff.toString());
+        ldapServer.startListening();
+        ldapPort = ldapServer.getListenPort();
+    }
+
+    private void createLdapBackendConf() throws IOException {
+        String backendConf = KdcConfigKey.KDC_IDENTITY_BACKEND.getPropertyKey()
+                + " = org.apache.kerby.kerberos.kdc.identitybackend.LdapIdentityBackend\n" + "host=127.0.0.1\n" + "port=" + ldapPort + "\n"
+                + "admin_dn=uid=admin,ou=system," + baseDn + "\n" + "admin_pw=secret\n" + "base_dn=" + baseDn;
+        Files.write(this.workDir.resolve("backend.conf"), backendConf.getBytes(StandardCharsets.UTF_8));
+        assert Files.exists(this.workDir.resolve("backend.conf"));
+    }
+
+    @SuppressForbidden(reason = "Uses Apache Kdc which requires usage of java.io.File in order to create a SimpleKdcServer")
+    private void prepareKdcServerAndStart() throws Exception {
+        // transport
+        simpleKdc.setWorkDir(workDir.toFile());
+        simpleKdc.setKdcHost(host);
+        simpleKdc.setKdcRealm(realm);
+        if (kdcPort == 0) {
+            kdcPort = NetworkUtil.getServerPort();
+        }
+        if (transport != null) {
+            if (transport.trim().equals("TCP")) {
+                simpleKdc.setKdcTcpPort(kdcPort);
+                simpleKdc.setAllowUdp(false);
+            } else if (transport.trim().equals("UDP")) {
+                simpleKdc.setKdcUdpPort(kdcPort);
+                simpleKdc.setAllowTcp(false);
+            } else {
+                throw new IllegalArgumentException("Invalid transport: " + transport);
+            }
+        } else {
+            throw new IllegalArgumentException("Need to set transport!");
+        }
+        final TimeValue minimumTicketLifeTime = new TimeValue(1, TimeUnit.DAYS);
+        final TimeValue maxRenewableLifeTime = new TimeValue(7, TimeUnit.DAYS);
+        simpleKdc.getKdcConfig().setLong(KdcConfigKey.MINIMUM_TICKET_LIFETIME, minimumTicketLifeTime.getMillis());
+        simpleKdc.getKdcConfig().setLong(KdcConfigKey.MAXIMUM_RENEWABLE_LIFETIME, maxRenewableLifeTime.getMillis());
+        simpleKdc.init();
+        simpleKdc.start();
+    }
+
+    public String getRealm() {
+        return realm;
+    }
+
+    public int getLdapListenPort() {
+        return ldapPort;
+    }
+
+    public int getKdcPort() {
+        return kdcPort;
+    }
+
+    /**
+     * Creates a principal in the KDC with the specified user and password.
+     *
+     * @param principal principal name, do not include the domain.
+     * @param password password.
+     * @throws Exception thrown if the principal could not be created.
+     */
+    public synchronized void createPrincipal(final String principal, final String password) throws Exception {
+        simpleKdc.createPrincipal(principal, password);
+    }
+
+    /**
+     * Creates multiple principals in the KDC and adds them to a keytab file.
+     *
+     * @param keytabFile keytab file to add the created principals. If keytab file
+     *            exists and then always appends to it.
+     * @param principals principals to add to the KDC, do not include the domain.
+     * @throws Exception thrown if the principals or the keytab file could not be
+     *             created.
+     */
+    @SuppressForbidden(reason = "Uses Apache Kdc which requires usage of java.io.File in order to create a SimpleKdcServer")
+    public synchronized void createPrincipal(final Path keytabFile, final String... principals) throws Exception {
+        simpleKdc.createPrincipals(principals);
+        for (String principal : principals) {
+            simpleKdc.getKadmin().exportKeytab(keytabFile.toFile(), principal);
+        }
+    }
+
+    /**
+     * Stop Simple Kdc Server
+     * 
+     * @throws PrivilegedActionException
+     */
+    public synchronized void stop() throws PrivilegedActionException {
+        AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
+
+            @Override
+            @SuppressForbidden(reason = "set or clear system property krb5 debug in kerberos tests")
+            public Void run() throws Exception {
+                if (simpleKdc != null) {
+                    try {
+                        simpleKdc.stop();
+                    } catch (KrbException e) {
+                        throw ExceptionsHelper.convertToRuntime(e);
+                    } finally {
+                        System.setProperty("sun.security.krb5.debug", Boolean.toString(krb5DebugBackupConfigValue));
+                    }
+                }
+
+                if (ldapServer != null) {
+                    ldapServer.shutDown(true);
+                }
+                return null;
+            }
+        });
+        logger.info("SimpleKdcServer stoppped.");
+    }
+
+}
\ No newline at end of file
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServerTests.java
new file mode 100644
index 00000000000..b1c75d957a7
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServerTests.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import com.unboundid.ldap.sdk.LDAPConnection;
+import com.unboundid.ldap.sdk.SearchResult;
+import com.unboundid.ldap.sdk.SearchScope;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidator;
+import org.elasticsearch.xpack.security.authc.ldap.support.LdapUtils;
+import org.ietf.jgss.GSSException;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.PrivilegedActionException;
+import java.text.ParseException;
+import java.util.Base64;
+
+import javax.security.auth.login.LoginException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class SimpleKdcLdapServerTests extends KerberosTestCase {
+
+    public void testPrincipalCreationAndSearchOnLdap() throws Exception {
+        simpleKdcLdapServer.createPrincipal(workDir.resolve("p1p2.keytab"), "p1", "p2");
+        assertTrue(Files.exists(workDir.resolve("p1p2.keytab")));
+        try (LDAPConnection ldapConn =
+                LdapUtils.privilegedConnect(() -> new LDAPConnection("localhost", simpleKdcLdapServer.getLdapListenPort()));) {
+            assertThat(ldapConn.isConnected(), is(true));
+            SearchResult sr = ldapConn.search("dc=example,dc=com", SearchScope.SUB, "(krb5PrincipalName=p1@EXAMPLE.COM)");
+            assertThat(sr.getSearchEntries(), hasSize(1));
+            assertThat(sr.getSearchEntries().get(0).getDN(), equalTo("uid=p1,dc=example,dc=com"));
+        }
+    }
+
+    public void testClientServiceMutualAuthentication() throws PrivilegedActionException, GSSException, LoginException, ParseException {
+        final String serviceUserName = randomFrom(serviceUserNames);
+        // Client login and init token preparation
+        final String clientUserName = randomFrom(clientUserNames);
+        try (SpnegoClient spnegoClient =
+                new SpnegoClient(principalName(clientUserName), new SecureString("pwd".toCharArray()), principalName(serviceUserName));) {
+            final String base64KerbToken = spnegoClient.getBase64EncodedTokenForSpnegoHeader();
+            assertThat(base64KerbToken, is(notNullValue()));
+            final KerberosAuthenticationToken kerbAuthnToken = new KerberosAuthenticationToken(Base64.getDecoder().decode(base64KerbToken));
+
+            // Service Login
+            final Environment env = TestEnvironment.newEnvironment(globalSettings);
+            final Path keytabPath = env.configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(settings));
+            // Handle Authz header which contains base64 token
+            final PlainActionFuture<Tuple<String, String>> future = new PlainActionFuture<>();
+            new KerberosTicketValidator().validateTicket((byte[]) kerbAuthnToken.credentials(), keytabPath, true, future);
+            assertThat(future.actionGet(), is(notNullValue()));
+            assertThat(future.actionGet().v1(), equalTo(principalName(clientUserName)));
+
+            // Authenticate service on client side.
+            final String outToken = spnegoClient.handleResponse(future.actionGet().v2());
+            assertThat(outToken, is(nullValue()));
+            assertThat(spnegoClient.isEstablished(), is(true));
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
new file mode 100644
index 00000000000..1f883b928bd
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
@@ -0,0 +1,257 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidator;
+import org.ietf.jgss.GSSContext;
+import org.ietf.jgss.GSSCredential;
+import org.ietf.jgss.GSSException;
+import org.ietf.jgss.GSSManager;
+import org.ietf.jgss.GSSName;
+
+import java.io.IOException;
+import java.security.AccessController;
+import java.security.Principal;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import javax.security.auth.Subject;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
+
+/**
+ * This class is used as a Spnego client during testing and handles SPNEGO
+ * interactions using GSS context negotiation.<br>
+ * It is not advisable to share a SpnegoClient between threads as there is no
+ * synchronization in place, internally this depends on {@link GSSContext} for
+ * context negotiation which maintains sequencing for replay detections.<br>
+ * Use {@link #close()} to release and dispose {@link LoginContext} and
+ * {@link GSSContext} after usage.
+ */
+class SpnegoClient implements AutoCloseable {
+    private static final Logger LOGGER = ESLoggerFactory.getLogger(SpnegoClient.class);
+
+    public static final String CRED_CONF_NAME = "PasswordConf";
+    private static final String SUN_KRB5_LOGIN_MODULE = "com.sun.security.auth.module.Krb5LoginModule";
+    private final GSSManager gssManager = GSSManager.getInstance();
+    private final LoginContext loginContext;
+    private final GSSContext gssContext;
+
+    /**
+     * Creates SpengoClient to interact with given service principal<br>
+     * Use {@link #close()} to logout {@link LoginContext} and dispose
+     * {@link GSSContext} after usage.
+     * @param userPrincipalName User principal name for login as client
+     * @param password password for client
+     * @param servicePrincipalName Service principal name with whom this client
+     *            interacts with.
+     * @throws PrivilegedActionException
+     * @throws GSSException
+     */
+    SpnegoClient(final String userPrincipalName, final SecureString password, final String servicePrincipalName)
+            throws PrivilegedActionException, GSSException {
+        String oldUseSubjectCredsOnlyFlag = null;
+        try {
+            oldUseSubjectCredsOnlyFlag = getAndSetUseSubjectCredsOnlySystemProperty("true");
+            LOGGER.info("SpnegoClient with userPrincipalName : {}", userPrincipalName);
+            final GSSName gssUserPrincipalName = gssManager.createName(userPrincipalName, GSSName.NT_USER_NAME);
+            final GSSName gssServicePrincipalName = gssManager.createName(servicePrincipalName, GSSName.NT_USER_NAME);
+            loginContext = AccessController
+                    .doPrivileged((PrivilegedExceptionAction<LoginContext>) () -> loginUsingPassword(userPrincipalName, password));
+            final GSSCredential userCreds = KerberosTestCase.doAsWrapper(loginContext.getSubject(),
+                    (PrivilegedExceptionAction<GSSCredential>) () -> gssManager.createCredential(gssUserPrincipalName,
+                            GSSCredential.DEFAULT_LIFETIME, KerberosTicketValidator.SPNEGO_OID, GSSCredential.INITIATE_ONLY));
+            gssContext = gssManager.createContext(gssServicePrincipalName.canonicalize(KerberosTicketValidator.SPNEGO_OID),
+                    KerberosTicketValidator.SPNEGO_OID, userCreds, GSSCredential.DEFAULT_LIFETIME);
+            gssContext.requestMutualAuth(true);
+        } catch (PrivilegedActionException pve) {
+            LOGGER.error("privileged action exception, with root cause", pve.getException());
+            throw pve;
+        } finally {
+            getAndSetUseSubjectCredsOnlySystemProperty(oldUseSubjectCredsOnlyFlag);
+        }
+    }
+
+    /**
+     * GSSContext initiator side handling, initiates context establishment and returns the
+     * base64 encoded token to be sent to server.
+     *
+     * @return Base64 encoded token
+     * @throws PrivilegedActionException
+     */
+    String getBase64EncodedTokenForSpnegoHeader() throws PrivilegedActionException {
+        final byte[] outToken = KerberosTestCase.doAsWrapper(loginContext.getSubject(),
+                (PrivilegedExceptionAction<byte[]>) () -> gssContext.initSecContext(new byte[0], 0, 0));
+        return Base64.getEncoder().encodeToString(outToken);
+    }
+
+    /**
+     * Handles server response and returns new token if any to be sent to server.
+     *
+     * @param base64Token inToken received from server passed to initSecContext for
+     *            gss negotiation
+     * @return Base64 encoded token to be sent to server. May return {@code null} if
+     *         nothing to be sent.
+     * @throws PrivilegedActionException
+     */
+    String handleResponse(final String base64Token) throws PrivilegedActionException {
+        if (gssContext.isEstablished()) {
+            throw new IllegalStateException("GSS Context has already been established");
+        }
+        final byte[] token = Base64.getDecoder().decode(base64Token);
+        final byte[] outToken = KerberosTestCase.doAsWrapper(loginContext.getSubject(),
+                (PrivilegedExceptionAction<byte[]>) () -> gssContext.initSecContext(token, 0, token.length));
+        if (outToken == null || outToken.length == 0) {
+            return null;
+        }
+        return Base64.getEncoder().encodeToString(outToken);
+    }
+
+    /**
+     * Spnego Client after usage needs to be closed in order to logout from
+     * {@link LoginContext} and dispose {@link GSSContext}
+     */
+    public void close() throws LoginException, GSSException, PrivilegedActionException {
+        if (loginContext != null) {
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                loginContext.logout();
+                return null;
+            });
+        }
+        if (gssContext != null) {
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                gssContext.dispose();
+                return null;
+            });
+        }
+    }
+
+    /**
+     * @return {@code true} If the gss security context was established
+     */
+    boolean isEstablished() {
+        return gssContext.isEstablished();
+    }
+
+    /**
+     * Performs authentication using provided principal name and password for client
+     *
+     * @param principal Principal name
+     * @param password {@link SecureString}
+     * @param settings {@link Settings}
+     * @return authenticated {@link LoginContext} instance. Note: This needs to be
+     *         closed {@link LoginContext#logout()} after usage.
+     * @throws LoginException
+     */
+    private static LoginContext loginUsingPassword(final String principal, final SecureString password) throws LoginException {
+        final Set<Principal> principals = Collections.singleton(new KerberosPrincipal(principal));
+
+        final Subject subject = new Subject(false, principals, Collections.emptySet(), Collections.emptySet());
+
+        final Configuration conf = new PasswordJaasConf(principal);
+        final CallbackHandler callback = new KrbCallbackHandler(principal, password);
+        final LoginContext loginContext = new LoginContext(CRED_CONF_NAME, subject, callback, conf);
+        loginContext.login();
+        return loginContext;
+    }
+
+    /**
+     * Usually we would have a JAAS configuration file for login configuration.
+     * Instead of an additional file setting as we do not want the options to be
+     * customizable we are constructing it in memory.
+     * <p>
+     * As we are uing this instead of jaas.conf, this requires refresh of
+     * {@link Configuration} and reqires appropriate security permissions to do so.
+     */
+    static class PasswordJaasConf extends Configuration {
+        private final String principal;
+
+        PasswordJaasConf(final String principal) {
+            this.principal = principal;
+        }
+
+        @Override
+        public AppConfigurationEntry[] getAppConfigurationEntry(final String name) {
+            final Map<String, String> options = new HashMap<>();
+            options.put("principal", principal);
+            options.put("storeKey", Boolean.TRUE.toString());
+            options.put("isInitiator", Boolean.TRUE.toString());
+            options.put("debug", Boolean.TRUE.toString());
+            // Refresh Krb5 config during tests as the port keeps changing for kdc server
+            options.put("refreshKrb5Config", Boolean.TRUE.toString());
+
+            return new AppConfigurationEntry[] { new AppConfigurationEntry(SUN_KRB5_LOGIN_MODULE,
+                    AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, Collections.unmodifiableMap(options)) };
+        }
+    }
+
+    /**
+     * Jaas call back handler to provide credentials.
+     */
+    static class KrbCallbackHandler implements CallbackHandler {
+        private final String principal;
+        private final SecureString password;
+
+        KrbCallbackHandler(final String principal, final SecureString password) {
+            this.principal = principal;
+            this.password = password;
+        }
+
+        public void handle(final Callback[] callbacks) throws IOException, UnsupportedCallbackException {
+            for (Callback callback : callbacks) {
+                if (callback instanceof PasswordCallback) {
+                    PasswordCallback pc = (PasswordCallback) callback;
+                    if (pc.getPrompt().contains(principal)) {
+                        pc.setPassword(password.getChars());
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    private static String getAndSetUseSubjectCredsOnlySystemProperty(final String value) {
+        String retVal = null;
+        try {
+            retVal = AccessController.doPrivileged(new PrivilegedExceptionAction<String>() {
+
+                @Override
+                @SuppressForbidden(
+                        reason = "For tests where we provide credentials, need to set and reset javax.security.auth.useSubjectCredsOnly")
+                public String run() throws Exception {
+                    String oldValue = System.getProperty("javax.security.auth.useSubjectCredsOnly");
+                    if (value != null) {
+                        System.setProperty("javax.security.auth.useSubjectCredsOnly", value);
+                    }
+                    return oldValue;
+                }
+
+            });
+        } catch (PrivilegedActionException e) {
+            throw ExceptionsHelper.convertToRuntime(e);
+        }
+        return retVal;
+    }
+}
diff --git a/x-pack/plugin/security/src/test/resources/kdc.ldiff b/x-pack/plugin/security/src/test/resources/kdc.ldiff
new file mode 100644
index 00000000000..e213048d6f5
--- /dev/null
+++ b/x-pack/plugin/security/src/test/resources/kdc.ldiff
@@ -0,0 +1,23 @@
+dn: dc=example,dc=com
+objectClass: top
+objectClass: domain
+dc: example
+
+dn: ou=system,dc=example,dc=com
+objectClass: organizationalUnit
+objectClass: top
+ou: system
+
+dn: ou=users,dc=example,dc=com
+objectClass: organizationalUnit
+objectClass: top
+ou: users
+
+dn: uid=admin,ou=system,dc=example,dc=com
+objectClass: top
+objectClass: person
+objectClass: inetOrgPerson
+cn: Admin
+sn: Admin
+uid: admin
+userPassword: secret
\ No newline at end of file
diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle
new file mode 100644
index 00000000000..5caf5d6947e
--- /dev/null
+++ b/x-pack/qa/kerberos-tests/build.gradle
@@ -0,0 +1,127 @@
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.Files
+
+apply plugin: 'elasticsearch.vagrantsupport'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+    testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
+}
+
+// MIT Kerberos Vagrant Testing Fixture
+String box = "krb5kdc"
+Map<String,String> vagrantEnvVars = [
+    'VAGRANT_CWD'           : "${project(':test:fixtures:krb5kdc-fixture').projectDir}",
+    'VAGRANT_VAGRANTFILE'   : 'Vagrantfile',
+    'VAGRANT_PROJECT_DIR'   : "${project(':test:fixtures:krb5kdc-fixture').projectDir}"
+]
+
+task krb5kdcUpdate(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'box'
+    subcommand 'update'
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn "vagrantCheckVersion", "virtualboxCheckVersion"
+}
+
+task krb5kdcFixture(type: org.elasticsearch.gradle.test.VagrantFixture) {
+    command 'up'
+    args '--provision', '--provider', 'virtualbox'
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcUpdate
+}
+
+task krb5AddPrincipals { dependsOn krb5kdcFixture }
+
+List<String> principals = [
+    "HTTP/localhost",
+    "peppa",
+    "george~dino"
+]
+String realm = "BUILD.ELASTIC.CO"
+
+for (String principal : principals) {
+    String[] princPwdPair = principal.split('~');
+    String princName = princPwdPair[0];
+    String password = "";
+    if (princPwdPair.length > 1) {
+        password = princPwdPair[1];
+    }
+    Task create = project.tasks.create("addPrincipal#${principal}".replace('/', '_'), org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+        command 'ssh'
+        args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $princName $password"
+        boxName box
+        environmentVars vagrantEnvVars
+        dependsOn krb5kdcFixture
+    }
+    krb5AddPrincipals.dependsOn(create)
+}
+
+def generatedResources = "$buildDir/generated-resources/keytabs"
+task copyKeytabToGeneratedResources(type: Copy) {
+    Path peppaKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("peppa.keytab").toAbsolutePath()
+    from peppaKeytab;
+    into generatedResources
+    dependsOn krb5AddPrincipals
+}
+
+integTestCluster {
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'xpack.security.enabled', 'true'
+    setting 'xpack.security.authc.realms.file.type', 'file'
+    setting 'xpack.security.authc.realms.file.order', '0'
+    setting 'xpack.ml.enabled', 'false'
+    setting 'xpack.security.audit.enabled', 'true'
+    // Kerberos realm
+    setting 'xpack.security.authc.realms.kerberos.type', 'kerberos'
+    setting 'xpack.security.authc.realms.kerberos.order', '1'
+    setting 'xpack.security.authc.realms.kerberos.keytab.path', 'es.keytab'
+    setting 'xpack.security.authc.realms.kerberos.krb.debug', 'true'
+    setting 'xpack.security.authc.realms.kerberos.remove_realm_name', 'false'
+
+    Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
+    String jvmArgsStr = " -Djava.security.krb5.conf=${krb5conf}" + " -Dsun.security.krb5.debug=true"
+    jvmArgs jvmArgsStr
+    Path esKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("HTTP_localhost.keytab").toAbsolutePath()
+    extraConfigFile("es.keytab", "${esKeytab}")
+
+    setupCommand 'setupTestAdmin',
+            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
+
+    waitCondition = { node, ant ->
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+        dest: tmpFile.toString(),
+        username: 'test_admin',
+        password: 'x-pack-test-password',
+        ignoreerrors: true,
+        retries: 10)
+        return tmpFile.exists()
+    }
+
+}
+
+integTestRunner {
+    Path peppaKeytab = Paths.get("${project.buildDir}", "generated-resources", "keytabs", "peppa.keytab")
+    systemProperty 'test.userkt', "peppa@${realm}"
+    systemProperty 'test.userkt.keytab', "${peppaKeytab}"
+    systemProperty 'test.userpwd', "george@${realm}"
+    systemProperty 'test.userpwd.password', "dino"
+    systemProperty 'tests.security.manager', 'true'
+    Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
+    List jvmargs = ["-Djava.security.krb5.conf=${krb5conf}","-Dsun.security.krb5.debug=true"]
+    jvmArgs jvmargs
+}
+
+if (project.rootProject.vagrantSupported == false) {
+    integTest.enabled = false
+} else {
+    project.sourceSets.test.output.dir(generatedResources, builtBy: copyKeytabToGeneratedResources)
+    integTestCluster.dependsOn krb5AddPrincipals, krb5kdcFixture, copyKeytabToGeneratedResources
+    integTest.finalizedBy project(':test:fixtures:krb5kdc-fixture').halt
+}
diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
new file mode 100644
index 00000000000..d5928cb58f6
--- /dev/null
+++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.security.AccessControlContext;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+import java.util.Map;
+
+import javax.security.auth.login.LoginContext;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+/**
+ * Integration test to demonstrate authentication against a real MIT Kerberos
+ * instance.
+ * <p>
+ * Demonstrates login by keytab and login by password for given user principal
+ * name using rest client.
+ */
+public class KerberosAuthenticationIT extends ESRestTestCase {
+    private static final String ENABLE_KERBEROS_DEBUG_LOGS_KEY = "test.krb.debug";
+    private static final String TEST_USER_WITH_KEYTAB_KEY = "test.userkt";
+    private static final String TEST_USER_WITH_KEYTAB_PATH_KEY = "test.userkt.keytab";
+    private static final String TEST_USER_WITH_PWD_KEY = "test.userpwd";
+    private static final String TEST_USER_WITH_PWD_PASSWD_KEY = "test.userpwd.password";
+    private static final String TEST_KERBEROS_REALM_NAME = "kerberos";
+
+    @Override
+    protected Settings restAdminSettings() {
+        final String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    /**
+     * Creates simple mapping that maps the users from 'kerberos' realm to
+     * the 'kerb_test' role.
+     */
+    @Before
+    public void setupRoleMapping() throws IOException {
+        final String json = Strings // top-level
+                .toString(XContentBuilder.builder(XContentType.JSON.xContent()).startObject()
+                        .array("roles", new String[] { "kerb_test" })
+                        .field("enabled", true)
+                        .startObject("rules")
+                        .startArray("all")
+                        .startObject().startObject("field").field("realm.name", TEST_KERBEROS_REALM_NAME).endObject().endObject()
+                        .endArray() // "all"
+                        .endObject() // "rules"
+                        .endObject());
+
+        final Request request = new Request("POST", "/_xpack/security/role_mapping/kerberosrolemapping");
+        request.setJsonEntity(json);
+        final Response response = adminClient().performRequest(request);
+        assertOK(response);
+    }
+
+    public void testLoginByKeytab() throws IOException, PrivilegedActionException {
+        final String userPrincipalName = System.getProperty(TEST_USER_WITH_KEYTAB_KEY);
+        final String keytabPath = System.getProperty(TEST_USER_WITH_KEYTAB_PATH_KEY);
+        final boolean enabledDebugLogs = Boolean.parseBoolean(System.getProperty(ENABLE_KERBEROS_DEBUG_LOGS_KEY));
+        final SpnegoHttpClientConfigCallbackHandler callbackHandler = new SpnegoHttpClientConfigCallbackHandler(userPrincipalName,
+                keytabPath, enabledDebugLogs);
+        executeRequestAndVerifyResponse(userPrincipalName, callbackHandler);
+    }
+
+    public void testLoginByUsernamePassword() throws IOException, PrivilegedActionException {
+        final String userPrincipalName = System.getProperty(TEST_USER_WITH_PWD_KEY);
+        final String password = System.getProperty(TEST_USER_WITH_PWD_PASSWD_KEY);
+        final boolean enabledDebugLogs = Boolean.parseBoolean(System.getProperty(ENABLE_KERBEROS_DEBUG_LOGS_KEY));
+        final SpnegoHttpClientConfigCallbackHandler callbackHandler = new SpnegoHttpClientConfigCallbackHandler(userPrincipalName,
+                new SecureString(password.toCharArray()), enabledDebugLogs);
+        executeRequestAndVerifyResponse(userPrincipalName, callbackHandler);
+    }
+
+    private void executeRequestAndVerifyResponse(final String userPrincipalName,
+            final SpnegoHttpClientConfigCallbackHandler callbackHandler) throws PrivilegedActionException, IOException {
+        final Request request = new Request("GET", "/_xpack/security/_authenticate");
+        try (RestClient restClient = buildRestClientForKerberos(callbackHandler)) {
+            final AccessControlContext accessControlContext = AccessController.getContext();
+            final LoginContext lc = callbackHandler.login();
+            Response response = SpnegoHttpClientConfigCallbackHandler.doAsPrivilegedWrapper(lc.getSubject(),
+                    (PrivilegedExceptionAction<Response>) () -> {
+                        return restClient.performRequest(request);
+                    }, accessControlContext);
+
+            assertOK(response);
+            final Map<String, Object> map = parseResponseAsMap(response.getEntity());
+            assertThat(map.get("username"), equalTo(userPrincipalName));
+            assertThat(map.get("roles"), instanceOf(List.class));
+            assertThat(((List<?>) map.get("roles")), contains("kerb_test"));
+        }
+    }
+
+    private Map<String, Object> parseResponseAsMap(final HttpEntity entity) throws IOException {
+        return convertToMap(XContentType.JSON.xContent(), entity.getContent(), false);
+    }
+
+    private RestClient buildRestClientForKerberos(final SpnegoHttpClientConfigCallbackHandler callbackHandler) throws IOException {
+        final Settings settings = restAdminSettings();
+        final HttpHost[] hosts = getClusterHosts().toArray(new HttpHost[getClusterHosts().size()]);
+
+        final RestClientBuilder restClientBuilder = RestClient.builder(hosts);
+        configureRestClientBuilder(restClientBuilder, settings);
+        restClientBuilder.setHttpClientConfigCallback(callbackHandler);
+        return restClientBuilder.build();
+    }
+
+    private static void configureRestClientBuilder(final RestClientBuilder restClientBuilder, final Settings settings)
+            throws IOException {
+        final String requestTimeoutString = settings.get(CLIENT_RETRY_TIMEOUT);
+        if (requestTimeoutString != null) {
+            final TimeValue maxRetryTimeout = TimeValue.parseTimeValue(requestTimeoutString, CLIENT_RETRY_TIMEOUT);
+            restClientBuilder.setMaxRetryTimeoutMillis(Math.toIntExact(maxRetryTimeout.getMillis()));
+        }
+        final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT);
+        if (socketTimeoutString != null) {
+            final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT);
+            restClientBuilder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis())));
+        }
+        if (settings.hasValue(CLIENT_PATH_PREFIX)) {
+            restClientBuilder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
+        }
+    }
+}
diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
new file mode 100644
index 00000000000..e5768d8f2e9
--- /dev/null
+++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
@@ -0,0 +1,317 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authc.kerberos;
+
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.KerberosCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.config.Lookup;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
+import org.elasticsearch.common.settings.SecureString;
+import org.ietf.jgss.GSSCredential;
+import org.ietf.jgss.GSSException;
+import org.ietf.jgss.GSSManager;
+import org.ietf.jgss.GSSName;
+import org.ietf.jgss.Oid;
+
+import java.io.IOException;
+import java.security.AccessControlContext;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.security.auth.Subject;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+import javax.security.auth.login.LoginContext;
+
+/**
+ * This class implements {@link HttpClientConfigCallback} which allows for
+ * customization of {@link HttpAsyncClientBuilder}.
+ * <p>
+ * Based on the configuration, configures {@link HttpAsyncClientBuilder} to
+ * support spengo auth scheme.<br>
+ * It uses configured credentials either password or keytab for authentication.
+ */
+public class SpnegoHttpClientConfigCallbackHandler implements HttpClientConfigCallback {
+    private static final String SUN_KRB5_LOGIN_MODULE = "com.sun.security.auth.module.Krb5LoginModule";
+    private static final String CRED_CONF_NAME = "ESClientLoginConf";
+    private static final Oid SPNEGO_OID = getSpnegoOid();
+
+    private static Oid getSpnegoOid() {
+        Oid oid = null;
+        try {
+            oid = new Oid("1.3.6.1.5.5.2");
+        } catch (GSSException gsse) {
+            throw ExceptionsHelper.convertToRuntime(gsse);
+        }
+        return oid;
+    }
+
+    private final String userPrincipalName;
+    private final SecureString password;
+    private final String keytabPath;
+    private final boolean enableDebugLogs;
+    private LoginContext loginContext;
+
+    /**
+     * Constructs {@link SpnegoHttpClientConfigCallbackHandler} with given
+     * principalName and password.
+     *
+     * @param userPrincipalName user principal name
+     * @param password password for user
+     * @param enableDebugLogs if {@code true} enables kerberos debug logs
+     */
+    public SpnegoHttpClientConfigCallbackHandler(final String userPrincipalName, final SecureString password,
+            final boolean enableDebugLogs) {
+        this.userPrincipalName = userPrincipalName;
+        this.password = password;
+        this.keytabPath = null;
+        this.enableDebugLogs = enableDebugLogs;
+    }
+
+    /**
+     * Constructs {@link SpnegoHttpClientConfigCallbackHandler} with given
+     * principalName and keytab.
+     *
+     * @param userPrincipalName User principal name
+     * @param keytabPath path to keytab file for user
+     * @param enableDebugLogs if {@code true} enables kerberos debug logs
+     */
+    public SpnegoHttpClientConfigCallbackHandler(final String userPrincipalName, final String keytabPath, final boolean enableDebugLogs) {
+        this.userPrincipalName = userPrincipalName;
+        this.keytabPath = keytabPath;
+        this.password = null;
+        this.enableDebugLogs = enableDebugLogs;
+    }
+
+    @Override
+    public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+        setupSpnegoAuthSchemeSupport(httpClientBuilder);
+        return httpClientBuilder;
+    }
+
+    private void setupSpnegoAuthSchemeSupport(HttpAsyncClientBuilder httpClientBuilder) {
+        final Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
+                .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build();
+
+        final GSSManager gssManager = GSSManager.getInstance();
+        try {
+            final GSSName gssUserPrincipalName = gssManager.createName(userPrincipalName, GSSName.NT_USER_NAME);
+            login();
+            final AccessControlContext acc = AccessController.getContext();
+            final GSSCredential credential = doAsPrivilegedWrapper(loginContext.getSubject(),
+                    (PrivilegedExceptionAction<GSSCredential>) () -> gssManager.createCredential(gssUserPrincipalName,
+                            GSSCredential.DEFAULT_LIFETIME, SPNEGO_OID, GSSCredential.INITIATE_ONLY),
+                    acc);
+
+            final KerberosCredentialsProvider credentialsProvider = new KerberosCredentialsProvider();
+            credentialsProvider.setCredentials(
+                    new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.SPNEGO),
+                    new KerberosCredentials(credential));
+            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+        } catch (GSSException e) {
+            throw new RuntimeException(e);
+        } catch (PrivilegedActionException e) {
+            throw new RuntimeException(e.getCause());
+        }
+        httpClientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
+    }
+
+    /**
+     * If logged in {@link LoginContext} is not available, it attempts login and
+     * returns {@link LoginContext}
+     *
+     * @return {@link LoginContext}
+     * @throws PrivilegedActionException
+     */
+    public synchronized LoginContext login() throws PrivilegedActionException {
+        if (this.loginContext == null) {
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                final Subject subject = new Subject(false, Collections.singleton(new KerberosPrincipal(userPrincipalName)),
+                        Collections.emptySet(), Collections.emptySet());
+                Configuration conf = null;
+                final CallbackHandler callback;
+                if (password != null) {
+                    conf = new PasswordJaasConf(userPrincipalName, enableDebugLogs);
+                    callback = new KrbCallbackHandler(userPrincipalName, password);
+                } else {
+                    conf = new KeytabJaasConf(userPrincipalName, keytabPath, enableDebugLogs);
+                    callback = null;
+                }
+                loginContext = new LoginContext(CRED_CONF_NAME, subject, callback, conf);
+                loginContext.login();
+                return null;
+            });
+        }
+        return loginContext;
+    }
+
+    /**
+     * Privileged Wrapper that invokes action with Subject.doAs to perform work as
+     * given subject.
+     *
+     * @param subject {@link Subject} to be used for this work
+     * @param action {@link PrivilegedExceptionAction} action for performing inside
+     *            Subject.doAs
+     * @param acc the {@link AccessControlContext} to be tied to the specified
+     *            subject and action see
+     *            {@link Subject#doAsPrivileged(Subject, PrivilegedExceptionAction, AccessControlContext)
+     * @return the value returned by the PrivilegedExceptionAction's run method
+     * @throws PrivilegedActionException
+     */
+    static <T> T doAsPrivilegedWrapper(final Subject subject, final PrivilegedExceptionAction<T> action, final AccessControlContext acc)
+            throws PrivilegedActionException {
+        try {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<T>) () -> Subject.doAsPrivileged(subject, action, acc));
+        } catch (PrivilegedActionException pae) {
+            if (pae.getCause() instanceof PrivilegedActionException) {
+                throw (PrivilegedActionException) pae.getCause();
+            }
+            throw pae;
+        }
+    }
+
+    /**
+     * This class matches {@link AuthScope} and based on that returns
+     * {@link Credentials}. Only supports {@link AuthSchemes#SPNEGO} in
+     * {@link AuthScope#getScheme()}
+     */
+    private static class KerberosCredentialsProvider implements CredentialsProvider {
+        private AuthScope authScope;
+        private Credentials credentials;
+
+        @Override
+        public void setCredentials(AuthScope authscope, Credentials credentials) {
+            if (authscope.getScheme().regionMatches(true, 0, AuthSchemes.SPNEGO, 0, AuthSchemes.SPNEGO.length()) == false) {
+                throw new IllegalArgumentException("Only " + AuthSchemes.SPNEGO + " auth scheme is supported in AuthScope");
+            }
+            this.authScope = authscope;
+            this.credentials = credentials;
+        }
+
+        @Override
+        public Credentials getCredentials(AuthScope authscope) {
+            assert this.authScope != null && authscope != null;
+            return authscope.match(this.authScope) > -1 ? this.credentials : null;
+        }
+
+        @Override
+        public void clear() {
+            this.authScope = null;
+            this.credentials = null;
+        }
+    }
+
+    /**
+     * Jaas call back handler to provide credentials.
+     */
+    private static class KrbCallbackHandler implements CallbackHandler {
+        private final String principal;
+        private final SecureString password;
+
+        KrbCallbackHandler(final String principal, final SecureString password) {
+            this.principal = principal;
+            this.password = password;
+        }
+
+        public void handle(final Callback[] callbacks) throws IOException, UnsupportedCallbackException {
+            for (Callback callback : callbacks) {
+                if (callback instanceof PasswordCallback) {
+                    PasswordCallback pc = (PasswordCallback) callback;
+                    if (pc.getPrompt().contains(principal)) {
+                        pc.setPassword(password.getChars());
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Usually we would have a JAAS configuration file for login configuration.
+     * Instead of an additional file setting as we do not want the options to be
+     * customizable we are constructing it in memory.
+     * <p>
+     * As we are using this instead of jaas.conf, this requires refresh of
+     * {@link Configuration} and reqires appropriate security permissions to do so.
+     */
+    private static class PasswordJaasConf extends AbstractJaasConf {
+
+        PasswordJaasConf(final String userPrincipalName, final boolean enableDebugLogs) {
+            super(userPrincipalName, enableDebugLogs);
+        }
+
+        public void addOptions(final Map<String, String> options) {
+            options.put("useTicketCache", Boolean.FALSE.toString());
+            options.put("useKeyTab", Boolean.FALSE.toString());
+        }
+    }
+
+    /**
+     * Usually we would have a JAAS configuration file for login configuration. As
+     * we have static configuration except debug flag, we are constructing in
+     * memory. This avoids additional configuration required from the user.
+     * <p>
+     * As we are using this instead of jaas.conf, this requires refresh of
+     * {@link Configuration} and requires appropriate security permissions to do so.
+     */
+    private static class KeytabJaasConf extends AbstractJaasConf {
+        private final String keytabFilePath;
+
+        KeytabJaasConf(final String userPrincipalName, final String keytabFilePath, final boolean enableDebugLogs) {
+            super(userPrincipalName, enableDebugLogs);
+            this.keytabFilePath = keytabFilePath;
+        }
+
+        public void addOptions(final Map<String, String> options) {
+            options.put("useKeyTab", Boolean.TRUE.toString());
+            options.put("keyTab", keytabFilePath);
+            options.put("doNotPrompt", Boolean.TRUE.toString());
+        }
+
+    }
+
+    private abstract static class AbstractJaasConf extends Configuration {
+        private final String userPrincipalName;
+        private final boolean enableDebugLogs;
+
+        AbstractJaasConf(final String userPrincipalName, final boolean enableDebugLogs) {
+            this.userPrincipalName = userPrincipalName;
+            this.enableDebugLogs = enableDebugLogs;
+        }
+
+        @Override
+        public AppConfigurationEntry[] getAppConfigurationEntry(final String name) {
+            final Map<String, String> options = new HashMap<>();
+            options.put("principal", userPrincipalName);
+            options.put("isInitiator", Boolean.TRUE.toString());
+            options.put("storeKey", Boolean.TRUE.toString());
+            options.put("debug", Boolean.toString(enableDebugLogs));
+            addOptions(options);
+            return new AppConfigurationEntry[] { new AppConfigurationEntry(SUN_KRB5_LOGIN_MODULE,
+                    AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, Collections.unmodifiableMap(options)) };
+        }
+
+        abstract void addOptions(Map<String, String> options);
+    }
+}
diff --git a/x-pack/qa/kerberos-tests/src/test/resources/plugin-security.policy b/x-pack/qa/kerberos-tests/src/test/resources/plugin-security.policy
new file mode 100644
index 00000000000..fb7936bf620
--- /dev/null
+++ b/x-pack/qa/kerberos-tests/src/test/resources/plugin-security.policy
@@ -0,0 +1,4 @@
+grant {
+  permission javax.security.auth.AuthPermission "doAsPrivileged";
+  permission javax.security.auth.kerberos.DelegationPermission "\"HTTP/localhost@BUILD.ELASTIC.CO\" \"krbtgt/BUILD.ELASTIC.CO@BUILD.ELASTIC.CO\"";
+};
\ No newline at end of file

From e6b9f59e4eb211c3c519864c3f195fa23c6fbc26 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 24 Jul 2018 11:53:04 -0400
Subject: [PATCH 157/260] Build: Shadow x-pack:protocol into x-pack:plugin:core
 (#32240)

This bundles the x-pack:protocol project into the x-pack:plugin:core
project because we'd like folks to consider it an implementation detail
of our build rather than a separate artifact to be managed and depended
on. It is now bundled into both x-pack:plugin:core and
client:rest-high-level. To make this work I had to fix a few things.

Firstly, I had to make PluginBuildPlugin work with the shadow plugin.
In that case we have to bundle only the `shadow` dependencies and the
shadow jar.

Secondly, every reference to x-pack:plugin:core has to use the `shadow`
configuration. Without that the reference is missing all of the
un-shadowed dependencies. I tried to make it so that applying the shadow
plugin automatically redefines the `default` configuration to mirror the
`shadow` configuration which would allow us to use bare project references
to the x-pack:plugin:core project but I couldn't make it work. It'd *look*
like it works but then fail for transitive dependencies anyway. I think
it is still a good thing to do but I don't have the willpower to do it
now.

Finally, I had to fix an issue where Eclipse and IntelliJ didn't properly
reference shadowed transitive dependencies. Neither IDE supports shadowing
natively so they have to reference the shadowed projects. We fix this by
detecting `shadow` dependencies when in "Intellij mode" or "Eclipse mode"
and adding `runtime` dependencies to the same target. This convinces
IntelliJ and Eclipse to play nice.
---
 CONTRIBUTING.md                               | 36 ++++++++++++++++++-
 build.gradle                                  | 25 +++++++++++++
 .../elasticsearch/gradle/BuildPlugin.groovy   | 12 +++++++
 .../gradle/plugin/PluginBuildPlugin.groovy    | 21 ++++-------
 x-pack/docs/build.gradle                      |  8 ++---
 x-pack/license-tools/build.gradle             |  4 +--
 x-pack/plugin/core/build.gradle               | 20 ++++++-----
 x-pack/plugin/deprecation/build.gradle        |  2 +-
 x-pack/plugin/graph/build.gradle              |  2 +-
 x-pack/plugin/logstash/build.gradle           |  2 +-
 x-pack/plugin/ml/build.gradle                 |  2 +-
 x-pack/plugin/monitoring/build.gradle         |  4 +--
 x-pack/plugin/rollup/build.gradle             |  2 +-
 x-pack/plugin/security/build.gradle           |  2 +-
 x-pack/plugin/security/cli/build.gradle       |  4 +--
 x-pack/plugin/sql/build.gradle                |  2 +-
 x-pack/plugin/upgrade/build.gradle            |  4 +--
 x-pack/plugin/watcher/build.gradle            |  2 +-
 x-pack/qa/full-cluster-restart/build.gradle   |  4 +--
 x-pack/qa/ml-basic-multi-node/build.gradle    |  2 +-
 x-pack/qa/ml-disabled/build.gradle            |  2 +-
 .../ml-native-multi-node-tests/build.gradle   |  2 +-
 x-pack/qa/ml-no-bootstrap-tests/build.gradle  |  3 +-
 x-pack/qa/ml-single-node-tests/build.gradle   |  2 +-
 .../build.gradle                              |  2 +-
 x-pack/qa/multi-node/build.gradle             |  2 +-
 x-pack/qa/openldap-tests/build.gradle         |  3 +-
 .../reindex-tests-with-security/build.gradle  |  2 +-
 x-pack/qa/rolling-upgrade-basic/build.gradle  |  2 +-
 x-pack/qa/rolling-upgrade/build.gradle        |  4 +--
 x-pack/qa/saml-idp-tests/build.gradle         |  3 +-
 x-pack/qa/security-client-tests/build.gradle  |  2 +-
 .../build.gradle                              |  2 +-
 x-pack/qa/security-migrate-tests/build.gradle |  2 +-
 .../build.gradle                              |  2 +-
 .../build.gradle                              |  2 +-
 .../smoke-test-ml-with-security/build.gradle  |  2 +-
 .../build.gradle                              |  3 +-
 x-pack/qa/smoke-test-plugins-ssl/build.gradle |  2 +-
 x-pack/qa/smoke-test-plugins/build.gradle     |  2 +-
 .../build.gradle                              |  2 +-
 .../build.gradle                              |  2 +-
 x-pack/qa/smoke-test-watcher/build.gradle     |  2 +-
 x-pack/qa/sql/security/build.gradle           |  4 +--
 x-pack/qa/third-party/hipchat/build.gradle    |  2 +-
 x-pack/qa/third-party/jira/build.gradle       |  2 +-
 x-pack/qa/third-party/pagerduty/build.gradle  |  2 +-
 x-pack/qa/third-party/slack/build.gradle      |  2 +-
 x-pack/qa/transport-client-tests/build.gradle |  2 +-
 x-pack/test/feature-aware/build.gradle        |  2 +-
 x-pack/transport-client/build.gradle          |  4 +--
 51 files changed, 147 insertions(+), 85 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b8f4262ca5c..8ee03c3d999 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -214,7 +214,7 @@ If your changes affect only the documentation, run:
 ```sh
 ./gradlew -p docs check
 ```
-For more information about testing code examples in the documentation, see 
+For more information about testing code examples in the documentation, see
 https://github.com/elastic/elasticsearch/blob/master/docs/README.asciidoc
 
 ### Project layout
@@ -305,6 +305,39 @@ the `qa` subdirectory functions just like the top level `qa` subdirectory. The
 Elasticsearch process. The `transport-client` subdirectory contains extensions
 to Elasticsearch's standard transport client to work properly with x-pack.
 
+### Gradle Build
+
+We use Gradle to build Elasticsearch because it is flexible enough to not only
+build and package Elasticsearch, but also orchestrate all of the ways that we
+have to test Elasticsearch.
+
+#### Configurations
+
+Gradle organizes dependencies and build artifacts into "configurations" and
+allows you to use these configurations arbitrarilly. Here are some of the most
+common configurations in our build and how we use them:
+
+<dl>
+<dt>`compile`</dt><dd>Code that is on the classpath at both compile and
+runtime. If the [`shadow`][shadow-plugin] plugin is applied to the project then
+this code is bundled into the jar produced by the project.</dd>
+<dt>`runtime`</dt><dd>Code that is not on the classpath at compile time but is
+on the classpath at runtime. We mostly use this configuration to make sure that
+we do not accidentally compile against dependencies of our dependencies also
+known as "transitive" dependencies".</dd>
+<dt>`compileOnly`</dt><dd>Code that is on the classpath at comile time but that
+should not be shipped with the project because it is "provided" by the runtime
+somehow. Elasticsearch plugins use this configuration to include dependencies
+that are bundled with Elasticsearch's server.</dd>
+<dt>`shadow`</dt><dd>Only available in projects with the shadow plugin. Code
+that is on the classpath at both compile and runtime but it *not* bundled into
+the jar produced by the project. If you depend on a project with the `shadow`
+plugin then you need to depend on this configuration because it will bring
+along all of the dependencies you need at runtime.</dd>
+<dt>`testCompile`</dt><dd>Code that is on the classpath for compiling tests
+that are part of this project but not production code. The canonical example
+of this is `junit`.</dd>
+</dl>
 
 Contributing as part of a class
 -------------------------------
@@ -337,3 +370,4 @@ repeating in this section because it has come up in this context.
 
 [eclipse]: http://www.eclipse.org/community/eclipse_newsletter/2017/june/
 [intellij]: https://blog.jetbrains.com/idea/2017/07/intellij-idea-2017-2-is-here-smart-sleek-and-snappy/
+[shadow-plugin]: https://github.com/johnrengelman/shadow
diff --git a/build.gradle b/build.gradle
index 66f34d8f445..8df89cb064c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -516,6 +516,31 @@ allprojects {
   tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
 }
 
+allprojects {
+  /*
+   * IntelliJ and Eclipse don't know about the shadow plugin so when we're
+   * in "IntelliJ mode" or "Eclipse mode" add "runtime" dependencies
+   * eveywhere where we see a "shadow" dependency which will cause them to
+   * reference shadowed projects directly rather than rely on the shadowing
+   * to include them. This is the correct thing for it to do because it
+   * doesn't run the jar shadowing at all. This isn't needed for the project
+   * itself because the IDE configuration is done by SourceSets but it is
+   * *is* needed for projects that depends on the project doing the shadowing.
+   * Without this they won't properly depend on the shadowed project.
+   */
+  if (isEclipse || isIdea) {
+	configurations.all { Configuration configuration ->
+	  dependencies.all { Dependency dep ->
+		if (dep instanceof ProjectDependency) {
+		  if (dep.getTargetConfiguration() == 'shadow') {
+			configuration.dependencies.add(project.dependencies.project(path: dep.dependencyProject.path, configuration: 'runtime'))
+		  }
+		}
+	  }
+	}
+  }
+}
+
 // we need to add the same --debug-jvm option as
 // the real RunTask has, so we can pass it through
 class Run extends DefaultTask {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 85216cf7336..94a8bf26177 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -391,6 +391,9 @@ class BuildPlugin implements Plugin<Project> {
         project.configurations.compile.dependencies.all(disableTransitiveDeps)
         project.configurations.testCompile.dependencies.all(disableTransitiveDeps)
         project.configurations.compileOnly.dependencies.all(disableTransitiveDeps)
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            project.configurations.shadow.dependencies.all(disableTransitiveDeps)
+        }
     }
 
     /** Adds repositories used by ES dependencies */
@@ -882,11 +885,20 @@ class BuildPlugin implements Plugin<Project> {
         project.dependencyLicenses.dependencies = project.configurations.runtime.fileCollection {
             it.group.startsWith('org.elasticsearch') == false
         } - project.configurations.compileOnly
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            project.dependencyLicenses.dependencies += project.configurations.shadow.fileCollection {
+                it.group.startsWith('org.elasticsearch') == false
+            }
+        }
     }
 
     private static configureDependenciesInfo(Project project) {
         Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class)
         deps.runtimeConfiguration = project.configurations.runtime
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            deps.runtimeConfiguration = project.configurations.create('infoDeps')
+            deps.runtimeConfiguration.extendsFrom(project.configurations.runtime, project.configurations.shadow)
+        }
         deps.compileOnlyConfiguration = project.configurations.compileOnly
         project.afterEvaluate {
             deps.mappings = project.dependencyLicenses.mappings
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index 7f6f337e8a9..00f178fda9c 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -48,18 +48,6 @@ public class PluginBuildPlugin extends BuildPlugin {
     @Override
     public void apply(Project project) {
         super.apply(project)
-        project.plugins.withType(ShadowPlugin).whenPluginAdded {
-            /*
-             * We've not tested these plugins together and we're fairly sure
-             * they aren't going to work properly as is *and* we're not really
-             * sure *why* you'd want to shade stuff in plugins. So we throw an
-             * exception here to make you come and read this comment. If you
-             * have a need for shadow while building plugins then know that you
-             * are probably going to have to fight with gradle for a while....
-             */
-            throw new InvalidUserDataException('elasticsearch.esplugin is not '
-                    + 'compatible with com.github.johnrengelman.shadow');
-        }
         configureDependencies(project)
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
@@ -153,8 +141,13 @@ public class PluginBuildPlugin extends BuildPlugin {
                 include(buildProperties.descriptorOutput.name)
             }
             from pluginMetadata // metadata (eg custom security policy)
-            from project.jar // this plugin's jar
-            from project.configurations.runtime - project.configurations.compileOnly // the dep jars
+            /*
+             * If the plugin is using the shadow plugin then we need to bundle
+             * "shadow" things rather than the default jar and dependencies so
+             * we don't hit jar hell.
+             */
+            from { project.plugins.hasPlugin(ShadowPlugin) ? project.shadowJar : project.jar }
+            from { project.plugins.hasPlugin(ShadowPlugin) ? project.configurations.shadow : project.configurations.runtime - project.configurations.compileOnly }
             // extra files for the plugin to go into the zip
             from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging
             from('src/main') {
diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle
index 4176cdeb0b7..2ae26044be5 100644
--- a/x-pack/docs/build.gradle
+++ b/x-pack/docs/build.gradle
@@ -30,7 +30,7 @@ buildRestTests.expectedUnconvertedCandidates = [
 ]
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
@@ -264,7 +264,7 @@ setups['farequote_index'] = '''
                   airline:
                      type: keyword
                   doc_count:
-                     type: integer       
+                     type: integer
 '''
 setups['farequote_data'] = setups['farequote_index'] + '''
   - do:
@@ -277,7 +277,7 @@ setups['farequote_data'] = setups['farequote_index'] + '''
             {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000", "doc_count": 5}
             {"index": {"_id":"2"}}
             {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000", "doc_count": 23}
-            {"index": {"_id":"3"}}            
+            {"index": {"_id":"3"}}
             {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000", "doc_count": 42}
 '''
 setups['farequote_job'] = setups['farequote_data'] + '''
@@ -309,7 +309,7 @@ setups['farequote_datafeed'] = setups['farequote_job'] + '''
           "job_id":"farequote",
           "indexes":"farequote"
           }
-'''          
+'''
 setups['server_metrics_index'] = '''
   - do:
         indices.create:
diff --git a/x-pack/license-tools/build.gradle b/x-pack/license-tools/build.gradle
index 3ef08073bbf..183b9ab50e0 100644
--- a/x-pack/license-tools/build.gradle
+++ b/x-pack/license-tools/build.gradle
@@ -1,7 +1,7 @@
 apply plugin: 'elasticsearch.build'
 
 dependencies {
-  compile project(xpackModule('core'))
+  compile project(path: xpackModule('core'), configuration: 'shadow')
   compile "org.elasticsearch:elasticsearch:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
 }
@@ -17,7 +17,7 @@ task buildZip(type: Zip, dependsOn: jar) {
   into(parentDir + '/lib') {
     from jar
     from configurations.runtime
-  } 
+  }
   into(parentDir + '/bin') {
     from 'bin'
   }
diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index 7ed98ccb5b4..b498246391d 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -8,6 +8,7 @@ import java.nio.file.StandardCopyOption
 apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
+apply plugin: 'com.github.johnrengelman.shadow'
 
 archivesBaseName = 'x-pack-core'
 
@@ -27,17 +28,17 @@ dependencyLicenses {
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${version}"
     compile project(':x-pack:protocol')
-    compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
-    compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
-    compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
-    compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
+    shadow "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+    shadow "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+    shadow "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
+    shadow "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
 
-    compile "commons-logging:commons-logging:${versions.commonslogging}"
-    compile "commons-codec:commons-codec:${versions.commonscodec}"
+    shadow "commons-logging:commons-logging:${versions.commonslogging}"
+    shadow "commons-codec:commons-codec:${versions.commonscodec}"
 
     // security deps
-    compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile project(path: ':modules:transport-netty4', configuration: 'runtime')
+    shadow 'com.unboundid:unboundid-ldapsdk:3.2.0'
+    shadow project(path: ':modules:transport-netty4', configuration: 'runtime')
 
     testCompile 'org.elasticsearch:securemock:1.2'
     testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
@@ -107,7 +108,8 @@ test {
 // TODO: don't publish test artifacts just to run messy tests, fix the tests!
 // https://github.com/elastic/x-plugins/issues/724
 configurations {
-    testArtifacts.extendsFrom testRuntime
+    testArtifacts.extendsFrom(testRuntime, shadow)
+    testArtifacts.exclude(group: project(':x-pack:protocol').group, module: project(':x-pack:protocol').name)
 }
 task testJar(type: Jar) {
     appendix 'test'
diff --git a/x-pack/plugin/deprecation/build.gradle b/x-pack/plugin/deprecation/build.gradle
index d89eb62e884..3746287d615 100644
--- a/x-pack/plugin/deprecation/build.gradle
+++ b/x-pack/plugin/deprecation/build.gradle
@@ -10,7 +10,7 @@ esplugin {
 archivesBaseName = 'x-pack-deprecation'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
 }
 
 run {
diff --git a/x-pack/plugin/graph/build.gradle b/x-pack/plugin/graph/build.gradle
index 3ddd2ebee04..2b0f592b720 100644
--- a/x-pack/plugin/graph/build.gradle
+++ b/x-pack/plugin/graph/build.gradle
@@ -10,7 +10,7 @@ esplugin {
 archivesBaseName = 'x-pack-graph'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
diff --git a/x-pack/plugin/logstash/build.gradle b/x-pack/plugin/logstash/build.gradle
index 5a9b5d6cbe5..2e158a90ac7 100644
--- a/x-pack/plugin/logstash/build.gradle
+++ b/x-pack/plugin/logstash/build.gradle
@@ -10,7 +10,7 @@ esplugin {
 archivesBaseName = 'x-pack-logstash'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 
 }
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index 4c3cc9eef93..a6730161337 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -40,7 +40,7 @@ compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,
 compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     // This should not be here
     testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
diff --git a/x-pack/plugin/monitoring/build.gradle b/x-pack/plugin/monitoring/build.gradle
index 3fde6cd8c37..a452ef09a20 100644
--- a/x-pack/plugin/monitoring/build.gradle
+++ b/x-pack/plugin/monitoring/build.gradle
@@ -13,7 +13,7 @@ esplugin {
 archivesBaseName = 'x-pack-monitoring'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 
     // monitoring deps
@@ -66,7 +66,7 @@ task internalClusterTest(type: RandomizedTestingTask,
   include '**/*IT.class'
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
-check.dependsOn internalClusterTest 
+check.dependsOn internalClusterTest
 internalClusterTest.mustRunAfter test
 
 // also add an "alias" task to make typing on the command line easier task icTest {
diff --git a/x-pack/plugin/rollup/build.gradle b/x-pack/plugin/rollup/build.gradle
index ff9c30ed9a9..649a89bc2cd 100644
--- a/x-pack/plugin/rollup/build.gradle
+++ b/x-pack/plugin/rollup/build.gradle
@@ -16,7 +16,7 @@ compileTestJava.options.compilerArgs << "-Xlint:-rawtypes"
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${version}"
 
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle
index 4b17bd47812..6db533bbecf 100644
--- a/x-pack/plugin/security/build.gradle
+++ b/x-pack/plugin/security/build.gradle
@@ -12,7 +12,7 @@ esplugin {
 archivesBaseName = 'x-pack-security'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     compileOnly project(path: ':modules:transport-netty4', configuration: 'runtime')
     compileOnly project(path: ':plugins:transport-nio', configuration: 'runtime')
 
diff --git a/x-pack/plugin/security/cli/build.gradle b/x-pack/plugin/security/cli/build.gradle
index 1799a2c7b81..19b86dd91c7 100644
--- a/x-pack/plugin/security/cli/build.gradle
+++ b/x-pack/plugin/security/cli/build.gradle
@@ -4,7 +4,7 @@ archivesBaseName = 'elasticsearch-security-cli'
 
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${version}"
-    compileOnly xpackProject('plugin:core')
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     compile 'org.bouncycastle:bcprov-jdk15on:1.59'
     compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
     testImplementation 'com.google.jimfs:jimfs:1.1'
@@ -17,4 +17,4 @@ dependencies {
 
 dependencyLicenses {
     mapping from: /bc.*/, to: 'bouncycastle'
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/build.gradle b/x-pack/plugin/sql/build.gradle
index 436837e85b4..039e78c1495 100644
--- a/x-pack/plugin/sql/build.gradle
+++ b/x-pack/plugin/sql/build.gradle
@@ -19,7 +19,7 @@ archivesBaseName = 'x-pack-sql'
 integTest.enabled = false
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     compileOnly(project(':modules:lang-painless')) {
         // exclude ASM to not affect featureAware task on Java 10+
         exclude group: "org.ow2.asm"
diff --git a/x-pack/plugin/upgrade/build.gradle b/x-pack/plugin/upgrade/build.gradle
index 8e65f87da30..f95cde7134c 100644
--- a/x-pack/plugin/upgrade/build.gradle
+++ b/x-pack/plugin/upgrade/build.gradle
@@ -14,7 +14,7 @@ esplugin {
 archivesBaseName = 'x-pack-upgrade'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
@@ -39,7 +39,7 @@ task internalClusterTest(type: RandomizedTestingTask,
   include '**/*IT.class'
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
-check.dependsOn internalClusterTest 
+check.dependsOn internalClusterTest
 internalClusterTest.mustRunAfter test
 
 // also add an "alias" task to make typing on the command line easier
diff --git a/x-pack/plugin/watcher/build.gradle b/x-pack/plugin/watcher/build.gradle
index 2b7b73d3796..a0feab67463 100644
--- a/x-pack/plugin/watcher/build.gradle
+++ b/x-pack/plugin/watcher/build.gradle
@@ -25,7 +25,7 @@ dependencyLicenses {
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${version}"
 
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     compileOnly project(path: ':modules:transport-netty4', configuration: 'runtime')
     compileOnly project(path: ':plugins:transport-nio', configuration: 'runtime')
 
diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle
index 5a6f0c9cc83..3cf29701206 100644
--- a/x-pack/qa/full-cluster-restart/build.gradle
+++ b/x-pack/qa/full-cluster-restart/build.gradle
@@ -11,7 +11,7 @@ apply plugin: 'elasticsearch.build'
 test.enabled = false
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile (project(path: xpackModule('security'), configuration: 'runtime')) {
         // Need to drop the guava dependency here or we get a conflict with watcher's guava dependency.
         // This is total #$%, but the solution is to get the SAML realm (which uses guava) out of security proper
@@ -249,7 +249,7 @@ subprojects {
   check.dependsOn(integTest)
 
   dependencies {
-      testCompile project(path: xpackModule('core'), configuration: 'runtime')
+      testCompile project(path: xpackModule('core'), configuration: 'shadow')
       testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
       testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
       testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
diff --git a/x-pack/qa/ml-basic-multi-node/build.gradle b/x-pack/qa/ml-basic-multi-node/build.gradle
index d396d38b223..3df77aadccb 100644
--- a/x-pack/qa/ml-basic-multi-node/build.gradle
+++ b/x-pack/qa/ml-basic-multi-node/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/ml-disabled/build.gradle b/x-pack/qa/ml-disabled/build.gradle
index 22a7dfc74ac..e914def3507 100644
--- a/x-pack/qa/ml-disabled/build.gradle
+++ b/x-pack/qa/ml-disabled/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/ml-native-multi-node-tests/build.gradle b/x-pack/qa/ml-native-multi-node-tests/build.gradle
index b8627db3068..b1893b20c46 100644
--- a/x-pack/qa/ml-native-multi-node-tests/build.gradle
+++ b/x-pack/qa/ml-native-multi-node-tests/build.gradle
@@ -4,7 +4,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
     testCompile project(path: xpackModule('ml'), configuration: 'testArtifacts')
diff --git a/x-pack/qa/ml-no-bootstrap-tests/build.gradle b/x-pack/qa/ml-no-bootstrap-tests/build.gradle
index cad5201a67b..7e252afa302 100644
--- a/x-pack/qa/ml-no-bootstrap-tests/build.gradle
+++ b/x-pack/qa/ml-no-bootstrap-tests/build.gradle
@@ -1,7 +1,6 @@
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
-
diff --git a/x-pack/qa/ml-single-node-tests/build.gradle b/x-pack/qa/ml-single-node-tests/build.gradle
index 9fd4a8d44d2..b62e37894b3 100644
--- a/x-pack/qa/ml-single-node-tests/build.gradle
+++ b/x-pack/qa/ml-single-node-tests/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle
index f5265466965..5d90f974762 100644
--- a/x-pack/qa/multi-cluster-search-security/build.gradle
+++ b/x-pack/qa/multi-cluster-search-security/build.gradle
@@ -3,7 +3,7 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
diff --git a/x-pack/qa/multi-node/build.gradle b/x-pack/qa/multi-node/build.gradle
index 69c7a696ff0..19729cf367e 100644
--- a/x-pack/qa/multi-node/build.gradle
+++ b/x-pack/qa/multi-node/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
 }
 
 integTestCluster {
diff --git a/x-pack/qa/openldap-tests/build.gradle b/x-pack/qa/openldap-tests/build.gradle
index 5c0399a1d9f..24cd6184afa 100644
--- a/x-pack/qa/openldap-tests/build.gradle
+++ b/x-pack/qa/openldap-tests/build.gradle
@@ -5,7 +5,7 @@ apply plugin: 'elasticsearch.standalone-test'
 apply plugin: 'elasticsearch.vagrantsupport'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
@@ -32,4 +32,3 @@ namingConventions {
     // integ tests use Tests instead of IT
     skipIntegTestInDisguise = true
 }
-
diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle
index ddf72f7d458..097d343b279 100644
--- a/x-pack/qa/reindex-tests-with-security/build.gradle
+++ b/x-pack/qa/reindex-tests-with-security/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
   testCompile project(path: ':modules:reindex')
diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle
index 6d5b250b460..21ac4414d86 100644
--- a/x-pack/qa/rolling-upgrade-basic/build.gradle
+++ b/x-pack/qa/rolling-upgrade-basic/build.gradle
@@ -7,7 +7,7 @@ import java.nio.charset.StandardCharsets
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') // to be moved in a later commit
 }
 
diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle
index f60cbe4b34c..b983caa8669 100644
--- a/x-pack/qa/rolling-upgrade/build.gradle
+++ b/x-pack/qa/rolling-upgrade/build.gradle
@@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.build'
 test.enabled = false
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('security'), configuration: 'runtime')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') // to be moved in a later commit
 }
@@ -284,7 +284,7 @@ subprojects {
   check.dependsOn(integTest)
 
   dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     testCompile project(path: xpackModule('watcher'))
   }
diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle
index ebb0d88991f..752ec6fb307 100644
--- a/x-pack/qa/saml-idp-tests/build.gradle
+++ b/x-pack/qa/saml-idp-tests/build.gradle
@@ -6,7 +6,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
     testCompile 'com.google.jimfs:jimfs:1.1'
@@ -84,4 +84,3 @@ thirdPartyAudit.excludes = [
   // missing
   'com.ibm.icu.lang.UCharacter'
 ]
-
diff --git a/x-pack/qa/security-client-tests/build.gradle b/x-pack/qa/security-client-tests/build.gradle
index 4e517f4d363..97945fb00ef 100644
--- a/x-pack/qa/security-client-tests/build.gradle
+++ b/x-pack/qa/security-client-tests/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle
index b2fac075cb3..7aeed3ad62d 100644
--- a/x-pack/qa/security-example-spi-extension/build.gradle
+++ b/x-pack/qa/security-example-spi-extension/build.gradle
@@ -8,7 +8,7 @@ esplugin {
 }
 
 dependencies {
-  compileOnly project(path: xpackModule('core'), configuration: 'runtime')
+  compileOnly project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/security-migrate-tests/build.gradle b/x-pack/qa/security-migrate-tests/build.gradle
index 7ccf6d2349b..3a8a0cf1005 100644
--- a/x-pack/qa/security-migrate-tests/build.gradle
+++ b/x-pack/qa/security-migrate-tests/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('security'), configuration: 'runtime')
   testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime')
 }
diff --git a/x-pack/qa/security-setup-password-tests/build.gradle b/x-pack/qa/security-setup-password-tests/build.gradle
index 16accc87a94..adb159acf6f 100644
--- a/x-pack/qa/security-setup-password-tests/build.gradle
+++ b/x-pack/qa/security-setup-password-tests/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('security'), configuration: 'runtime')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
diff --git a/x-pack/qa/smoke-test-graph-with-security/build.gradle b/x-pack/qa/smoke-test-graph-with-security/build.gradle
index d3f788d0b06..9cdfaffccfb 100644
--- a/x-pack/qa/smoke-test-graph-with-security/build.gradle
+++ b/x-pack/qa/smoke-test-graph-with-security/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
 }
 
 // bring in graph rest test suite
diff --git a/x-pack/qa/smoke-test-ml-with-security/build.gradle b/x-pack/qa/smoke-test-ml-with-security/build.gradle
index 2a12aa2f28d..84c23add254 100644
--- a/x-pack/qa/smoke-test-ml-with-security/build.gradle
+++ b/x-pack/qa/smoke-test-ml-with-security/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
   testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
index 4e079430562..8ce0cde7657 100644
--- a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
+++ b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'))
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('watcher'))
   testCompile project(path: xpackModule('monitoring'))
 }
@@ -24,4 +24,3 @@ integTestCluster {
   // one of the exporters should configure cluster alerts
   // setting 'xpack.monitoring.exporters.my_http.cluster_alerts.management.enabled', 'true'
 }
-
diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
index 672bc3a19e8..53533bd9b87 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle
+++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
@@ -15,7 +15,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
 }
 
 String outputDir = "${buildDir}/generated-resources/${project.name}"
diff --git a/x-pack/qa/smoke-test-plugins/build.gradle b/x-pack/qa/smoke-test-plugins/build.gradle
index 4badc9d3509..b66903af18b 100644
--- a/x-pack/qa/smoke-test-plugins/build.gradle
+++ b/x-pack/qa/smoke-test-plugins/build.gradle
@@ -4,7 +4,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
 }
 
 ext.pluginsCount = 0
diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle
index 1c43db0b63e..d921c5f5b66 100644
--- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle
+++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
   testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
 }
diff --git a/x-pack/qa/smoke-test-watcher-with-security/build.gradle b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
index 0f052074bfb..a843641be80 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/build.gradle
+++ b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
 }
 
 // bring in watcher rest test suite
diff --git a/x-pack/qa/smoke-test-watcher/build.gradle b/x-pack/qa/smoke-test-watcher/build.gradle
index abfd27e729b..dc87248df61 100644
--- a/x-pack/qa/smoke-test-watcher/build.gradle
+++ b/x-pack/qa/smoke-test-watcher/build.gradle
@@ -7,7 +7,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
   testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
   testCompile project(path: ':modules:lang-painless', configuration: 'runtime')
diff --git a/x-pack/qa/sql/security/build.gradle b/x-pack/qa/sql/security/build.gradle
index 15f7734f942..f02886f80a1 100644
--- a/x-pack/qa/sql/security/build.gradle
+++ b/x-pack/qa/sql/security/build.gradle
@@ -1,5 +1,5 @@
 dependencies {
-  testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
 }
 
 Project mainProject = project
@@ -20,7 +20,7 @@ subprojects {
   }
 
   dependencies {
-    testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
   }
 
   integTestCluster {
diff --git a/x-pack/qa/third-party/hipchat/build.gradle b/x-pack/qa/third-party/hipchat/build.gradle
index cd37d6e738e..03b6c319698 100644
--- a/x-pack/qa/third-party/hipchat/build.gradle
+++ b/x-pack/qa/third-party/hipchat/build.gradle
@@ -4,7 +4,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle
index 078fed4dd36..3814c8e9a53 100644
--- a/x-pack/qa/third-party/jira/build.gradle
+++ b/x-pack/qa/third-party/jira/build.gradle
@@ -7,7 +7,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle
index 683e18caa1c..c0f337e160e 100644
--- a/x-pack/qa/third-party/pagerduty/build.gradle
+++ b/x-pack/qa/third-party/pagerduty/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle
index abcdad0e096..431752765f3 100644
--- a/x-pack/qa/third-party/slack/build.gradle
+++ b/x-pack/qa/third-party/slack/build.gradle
@@ -5,7 +5,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
diff --git a/x-pack/qa/transport-client-tests/build.gradle b/x-pack/qa/transport-client-tests/build.gradle
index c864a9084cb..a94ad8fd592 100644
--- a/x-pack/qa/transport-client-tests/build.gradle
+++ b/x-pack/qa/transport-client-tests/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime')
 }
 
diff --git a/x-pack/test/feature-aware/build.gradle b/x-pack/test/feature-aware/build.gradle
index 11b0e67183c..f6a1f6cb16f 100644
--- a/x-pack/test/feature-aware/build.gradle
+++ b/x-pack/test/feature-aware/build.gradle
@@ -3,7 +3,7 @@ apply plugin: 'elasticsearch.build'
 dependencies {
   compile 'org.ow2.asm:asm:6.2'
   compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch.plugin:x-pack-core:${version}"
+  compile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile "org.elasticsearch.test:framework:${version}"
 }
 
diff --git a/x-pack/transport-client/build.gradle b/x-pack/transport-client/build.gradle
index 31c05569274..7155dad5ee6 100644
--- a/x-pack/transport-client/build.gradle
+++ b/x-pack/transport-client/build.gradle
@@ -9,8 +9,8 @@ archivesBaseName = 'x-pack-transport'
 
 dependencies {
     // this "api" dependency looks weird, but it is correct, as it contains
-    // all of x-pack for now, and transport client will be going away in the future. 
-    compile "org.elasticsearch.plugin:x-pack-core:${version}"
+    // all of x-pack for now, and transport client will be going away in the future.
+    compile project(path: xpackModule('core'), configuration: 'shadow')
     compile "org.elasticsearch.client:transport:${version}"
     testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
     testCompile "junit:junit:${versions.junit}"

From 387c3c7f1d9c5aa6e5c02a71ed3a64ecf6c11872 Mon Sep 17 00:00:00 2001
From: Tim Vernum <tim@adjective.org>
Date: Wed, 25 Jul 2018 02:34:46 +1000
Subject: [PATCH 158/260] Introduce Application Privileges with support for
 Kibana RBAC (#32309)

This commit introduces "Application Privileges" to the X-Pack security
model.

Application Privileges are managed within Elasticsearch, and can be
tested with the _has_privileges API, but do not grant access to any
actions or resources within Elasticsearch. Their purpose is to allow
applications outside of Elasticsearch to represent and store their own
privileges model within Elasticsearch roles.

Access to manage application privileges is handled in a new way that
grants permission to specific application names only. This lays the
foundation for more OLS on cluster privileges, which is implemented by
allowing a cluster permission to inspect not just the action being
executed, but also the request to which the action is applied.
To support this, a "conditional cluster privilege" is introduced, which
is like the existing cluster privilege, except that it has a Predicate
over the request as well as over the action name.

Specifically, this adds
- GET/PUT/DELETE actions for defining application level privileges
- application privileges in role definitions
- application privileges in the has_privileges API
- changes to the cluster permission class to support checking of request
  objects
- a new "global" element on role definition to provide cluster object
  level security (only for manage application privileges)
- changes to `kibana_user`, `kibana_dashboard_only_user` and
  `kibana_system` roles to use and manage application privileges

Closes #29820
Closes #31559
---
 .../common/io/stream/StreamInput.java         |  18 +-
 .../common/io/stream/StreamOutput.java        |  11 +
 .../common/io/stream/StreamTests.java         |  48 +-
 .../org/elasticsearch/test/ESTestCase.java    |  15 +
 .../en/rest-api/security/privileges.asciidoc  |   3 +-
 .../docs/en/rest-api/security/roles.asciidoc  |   1 +
 .../xpack/core/XPackClientPlugin.java         |   7 +
 .../ApplicationPrivilegesRequest.java         |  17 +
 .../privilege/DeletePrivilegesAction.java     |  26 +
 .../privilege/DeletePrivilegesRequest.java    | 101 ++++
 .../DeletePrivilegesRequestBuilder.java       |  31 +
 .../privilege/DeletePrivilegesResponse.java   |  57 ++
 .../action/privilege/GetPrivilegesAction.java |  26 +
 .../privilege/GetPrivilegesRequest.java       |  78 +++
 .../GetPrivilegesRequestBuilder.java          |  29 +
 .../privilege/GetPrivilegesResponse.java      |  47 ++
 .../action/privilege/PutPrivilegesAction.java |  26 +
 .../privilege/PutPrivilegesRequest.java       | 124 ++++
 .../PutPrivilegesRequestBuilder.java          | 131 +++++
 .../privilege/PutPrivilegesResponse.java      |  60 ++
 .../security/action/role/PutRoleRequest.java  |  59 +-
 .../action/role/PutRoleRequestBuilder.java    |   2 +
 .../action/user/HasPrivilegesRequest.java     |  36 +-
 .../user/HasPrivilegesRequestBuilder.java     |   1 +
 .../action/user/HasPrivilegesResponse.java    |  78 ++-
 .../core/security/authz/RoleDescriptor.java   | 301 +++++++++-
 .../permission/ApplicationPermission.java     | 110 ++++
 .../authz/permission/ClusterPermission.java   |  85 ++-
 .../core/security/authz/permission/Role.java  |  80 ++-
 .../authz/privilege/ApplicationPrivilege.java | 225 ++++++++
 .../ApplicationPrivilegeDescriptor.java       | 194 +++++++
 .../ConditionalClusterPrivilege.java          |  63 ++
 .../ConditionalClusterPrivileges.java         | 225 ++++++++
 .../authz/store/ReservedRolesStore.java       |  35 +-
 .../core/security/client/SecurityClient.java  |  41 +-
 .../resources/security-index-template.json    |  38 ++
 .../org/elasticsearch/test/TestMatchers.java  |  26 +-
 .../DeletePrivilegesRequestTests.java         |  62 ++
 .../DeletePrivilegesResponseTests.java        |  30 +
 .../privilege/GetPrivilegesRequestTests.java  |  58 ++
 .../privilege/GetPrivilegesResponseTests.java |  41 ++
 .../privilege/PutPrivilegesRequestTests.java  |  97 ++++
 .../privilege/PutPrivilegesResponseTests.java |  42 ++
 .../action/role/PutRoleRequestTests.java      | 164 ++++++
 .../user/HasPrivilegesRequestTests.java       | 125 ++++
 .../ApplicationPermissionTests.java           | 124 ++++
 .../ApplicationPrivilegeDescriptorTests.java  | 143 +++++
 .../privilege/ApplicationPrivilegeTests.java  | 202 +++++++
 .../ConditionalClusterPrivilegesTests.java    |  73 +++
 .../ManageApplicationPrivilegesTests.java     | 156 +++++
 .../authz/store/ReservedRolesStoreTests.java  | 454 +++++++++------
 .../ml/action/TransportPutDatafeedAction.java |   5 +-
 .../xpack/security/Security.java              |  27 +-
 .../TransportDeletePrivilegesAction.java      |  52 ++
 .../TransportGetPrivilegesAction.java         |  56 ++
 .../TransportPutPrivilegesAction.java         |  47 ++
 .../user/TransportHasPrivilegesAction.java    | 101 +++-
 .../security/authz/AuthorizationService.java  |   4 +-
 .../authz/store/CompositeRolesStore.java      | 103 +++-
 .../authz/store/NativePrivilegeStore.java     | 278 +++++++++
 .../authz/store/NativeRolesStore.java         |  15 +-
 .../rest/action/SecurityBaseRestHandler.java  |   2 +-
 .../privilege/RestDeletePrivilegesAction.java |  66 +++
 .../privilege/RestGetPrivilegesAction.java    |  91 +++
 .../privilege/RestPutPrivilegeAction.java     |  49 ++
 .../privilege/RestPutPrivilegesAction.java    |  78 +++
 .../action/user/RestHasPrivilegesAction.java  |  29 +-
 .../KibanaSystemRoleIntegTests.java           |  66 +++
 .../integration/KibanaUserRoleIntegTests.java |  28 +-
 .../PutPrivilegesRequestBuilderTests.java     | 127 ++++
 .../HasPrivilegesRequestBuilderTests.java     |   2 +-
 .../TransportHasPrivilegesActionTests.java    | 248 +++++++-
 .../security/audit/index/AuditTrailTests.java |   2 +-
 .../ESNativeRealmMigrateToolTests.java        |  10 +-
 .../authc/esnative/NativeRealmIntegTests.java |   7 +-
 .../authz/AuthorizationServiceTests.java      | 546 ++++++++++--------
 .../authz/AuthorizedIndicesTests.java         |   8 +-
 .../authz/IndicesAndAliasesResolverTests.java |   5 +-
 .../security/authz/RoleDescriptorTests.java   | 121 +++-
 .../authz/store/CompositeRolesStoreTests.java | 154 ++++-
 .../authz/store/FileRolesStoreTests.java      |  18 +-
 .../store/NativePrivilegeStoreTests.java      | 300 ++++++++++
 .../user/HasPrivilegesRestResponseTests.java  |  18 +-
 .../api/xpack.security.delete_privileges.json |  30 +
 .../api/xpack.security.get_privileges.json    |  24 +
 .../api/xpack.security.has_privileges.json    |  22 +
 .../api/xpack.security.put_privilege.json     |  33 ++
 .../api/xpack.security.put_privileges.json    |  27 +
 .../test/privileges/10_basic.yml              | 324 +++++++++++
 .../privileges/20_has_application_privs.yml   | 190 ++++++
 .../test/privileges/30_superuser.yml          | 131 +++++
 .../test/roles/40_global_privileges.yml       |  51 ++
 .../authz/40_condtional_cluster_priv.yml      | 132 +++++
 93 files changed, 7171 insertions(+), 682 deletions(-)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ApplicationPrivilegesRequest.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequest.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesResponse.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestBuilder.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponse.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequest.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesResponse.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilege.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeDescriptor.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivilege.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermissionTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeDescriptorTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivilegesTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportDeletePrivilegesAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegeAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaSystemRoleIntegTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_privileges.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_privileges.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.has_privileges.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privilege.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privileges.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/10_basic.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/20_has_application_privs.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/30_superuser.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/roles/40_global_privileges.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/40_condtional_cluster_priv.yml

diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index d7879b0d928..fd9ffdfd31d 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -59,6 +59,7 @@ import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.EnumSet;
@@ -931,8 +932,23 @@ public abstract class StreamInput extends InputStream {
      * Reads a list of objects
      */
     public <T> List<T> readList(Writeable.Reader<T> reader) throws IOException {
+        return readCollection(reader, ArrayList::new);
+    }
+
+    /**
+     * Reads a set of objects
+     */
+    public <T> Set<T> readSet(Writeable.Reader<T> reader) throws IOException {
+        return readCollection(reader, HashSet::new);
+    }
+
+    /**
+     * Reads a collection of objects
+     */
+    private <T, C extends Collection<? super T>> C readCollection(Writeable.Reader<T> reader,
+                                                                  IntFunction<C> constructor) throws IOException {
         int count = readArraySize();
-        List<T> builder = new ArrayList<>(count);
+        C builder = constructor.apply(count);
         for (int i=0; i<count; i++) {
             builder.add(reader.read(this));
         }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index e8c4d197fda..199b8872c55 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -55,6 +55,7 @@ import java.nio.file.FileSystemLoopException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.NotDirectoryException;
 import java.time.ZonedDateTime;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.EnumMap;
@@ -995,6 +996,16 @@ public abstract class StreamOutput extends OutputStream {
         }
     }
 
+    /**
+     * Writes a collection of generic objects via a {@link Writer}
+     */
+    public <T> void writeCollection(Collection<T> collection, Writer<T> writer) throws IOException {
+        writeVInt(collection.size());
+        for (T val: collection) {
+            writer.write(this, val);
+        }
+    }
+
     /**
      * Writes a list of strings
      */
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
index d64dece7867..6431a3469b6 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
@@ -31,6 +31,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -42,6 +43,7 @@ import java.util.stream.IntStream;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.iterableWithSize;
 
 public class StreamTests extends ESTestCase {
 
@@ -65,7 +67,7 @@ public class StreamTests extends ESTestCase {
         final Set<Byte> set = IntStream.range(Byte.MIN_VALUE, Byte.MAX_VALUE).mapToObj(v -> (byte) v).collect(Collectors.toSet());
         set.remove((byte) 0);
         set.remove((byte) 1);
-        final byte[] corruptBytes = new byte[] { randomFrom(set) };
+        final byte[] corruptBytes = new byte[]{randomFrom(set)};
         final BytesReference corrupt = new BytesArray(corruptBytes);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> corrupt.streamInput().readBoolean());
         final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", corruptBytes[0]);
@@ -100,7 +102,7 @@ public class StreamTests extends ESTestCase {
         set.remove((byte) 0);
         set.remove((byte) 1);
         set.remove((byte) 2);
-        final byte[] corruptBytes = new byte[] { randomFrom(set) };
+        final byte[] corruptBytes = new byte[]{randomFrom(set)};
         final BytesReference corrupt = new BytesArray(corruptBytes);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> corrupt.streamInput().readOptionalBoolean());
         final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", corruptBytes[0]);
@@ -119,22 +121,22 @@ public class StreamTests extends ESTestCase {
 
     public void testSpecificVLongSerialization() throws IOException {
         List<Tuple<Long, byte[]>> values =
-                Arrays.asList(
-                        new Tuple<>(0L, new byte[]{0}),
-                        new Tuple<>(-1L, new byte[]{1}),
-                        new Tuple<>(1L, new byte[]{2}),
-                        new Tuple<>(-2L, new byte[]{3}),
-                        new Tuple<>(2L, new byte[]{4}),
-                        new Tuple<>(Long.MIN_VALUE, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1, 1}),
-                        new Tuple<>(Long.MAX_VALUE, new byte[]{-2, -1, -1, -1, -1, -1, -1, -1, -1, 1})
+            Arrays.asList(
+                new Tuple<>(0L, new byte[]{0}),
+                new Tuple<>(-1L, new byte[]{1}),
+                new Tuple<>(1L, new byte[]{2}),
+                new Tuple<>(-2L, new byte[]{3}),
+                new Tuple<>(2L, new byte[]{4}),
+                new Tuple<>(Long.MIN_VALUE, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1, 1}),
+                new Tuple<>(Long.MAX_VALUE, new byte[]{-2, -1, -1, -1, -1, -1, -1, -1, -1, 1})
 
-                );
+            );
         for (Tuple<Long, byte[]> value : values) {
             BytesStreamOutput out = new BytesStreamOutput();
             out.writeZLong(value.v1());
             assertArrayEquals(Long.toString(value.v1()), value.v2(), BytesReference.toBytes(out.bytes()));
             BytesReference bytes = new BytesArray(value.v2());
-            assertEquals(Arrays.toString(value.v2()), (long)value.v1(), bytes.streamInput().readZLong());
+            assertEquals(Arrays.toString(value.v2()), (long) value.v1(), bytes.streamInput().readZLong());
         }
     }
 
@@ -158,7 +160,7 @@ public class StreamTests extends ESTestCase {
         }
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeGenericValue(write);
-        LinkedHashMap<String, Integer> read = (LinkedHashMap<String, Integer>)out.bytes().streamInput().readGenericValue();
+        LinkedHashMap<String, Integer> read = (LinkedHashMap<String, Integer>) out.bytes().streamInput().readGenericValue();
         assertEquals(size, read.size());
         int index = 0;
         for (Map.Entry<String, Integer> entry : read.entrySet()) {
@@ -172,7 +174,8 @@ public class StreamTests extends ESTestCase {
         final int length = randomIntBetween(1, 1024);
         StreamInput delegate = StreamInput.wrap(new byte[length]);
 
-        FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {};
+        FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {
+        };
         assertEquals(filterInputStream.available(), length);
 
         // read some bytes
@@ -201,7 +204,7 @@ public class StreamTests extends ESTestCase {
         }
         stream.writeByteArray(array);
         InputStreamStreamInput streamInput = new InputStreamStreamInput(StreamInput.wrap(BytesReference.toBytes(stream.bytes())), array
-            .length-1);
+            .length - 1);
         expectThrows(EOFException.class, streamInput::readByteArray);
         streamInput = new InputStreamStreamInput(StreamInput.wrap(BytesReference.toBytes(stream.bytes())), BytesReference.toBytes(stream
             .bytes()).length);
@@ -230,6 +233,21 @@ public class StreamTests extends ESTestCase {
         assertThat(targetArray, equalTo(sourceArray));
     }
 
+    public void testSetOfLongs() throws IOException {
+        final int size = randomIntBetween(0, 6);
+        final Set<Long> sourceSet = new HashSet<>(size);
+        for (int i = 0; i < size; i++) {
+            sourceSet.add(randomLongBetween(i * 1000, (i + 1) * 1000 - 1));
+        }
+        assertThat(sourceSet, iterableWithSize(size));
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.writeCollection(sourceSet, StreamOutput::writeLong);
+
+        final Set<Long> targetSet = out.bytes().streamInput().readSet(StreamInput::readLong);
+        assertThat(targetSet, equalTo(sourceSet));
+    }
+
     static final class WriteableString implements Writeable {
         final String string;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 13540eaec95..44fe621dfa4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -149,6 +149,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
+import java.util.function.IntFunction;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -717,6 +718,20 @@ public abstract class ESTestCase extends LuceneTestCase {
         return generateRandomStringArray(maxArraySize, stringSize, allowNull, true);
     }
 
+    public static <T> T[] randomArray(int maxArraySize, IntFunction<T[]> arrayConstructor, Supplier<T> valueConstructor) {
+        return randomArray(0, maxArraySize, arrayConstructor, valueConstructor);
+    }
+
+    public static <T> T[] randomArray(int minArraySize, int maxArraySize, IntFunction<T[]> arrayConstructor, Supplier<T> valueConstructor) {
+        final int size = randomIntBetween(minArraySize, maxArraySize);
+        final T[] array = arrayConstructor.apply(size);
+        for (int i = 0; i < array.length; i++) {
+            array[i] = valueConstructor.get();
+        }
+        return array;
+    }
+
+
     private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m", "micros", "nanos"};
 
     public static String randomTimeValue(int lower, int upper, String... suffixes) {
diff --git a/x-pack/docs/en/rest-api/security/privileges.asciidoc b/x-pack/docs/en/rest-api/security/privileges.asciidoc
index 4ec192d633b..adaf27e9707 100644
--- a/x-pack/docs/en/rest-api/security/privileges.asciidoc
+++ b/x-pack/docs/en/rest-api/security/privileges.asciidoc
@@ -84,7 +84,8 @@ The following example output indicates which privileges the "rdeniro" user has:
       "read" : true,
       "write" : false
     }
-  }
+  },
+  "application" : {}
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"rdeniro"/"$body.username"/]
diff --git a/x-pack/docs/en/rest-api/security/roles.asciidoc b/x-pack/docs/en/rest-api/security/roles.asciidoc
index b7b2260a0e4..28c09c560ec 100644
--- a/x-pack/docs/en/rest-api/security/roles.asciidoc
+++ b/x-pack/docs/en/rest-api/security/roles.asciidoc
@@ -140,6 +140,7 @@ role. If the role is not defined in the `native` realm, the request 404s.
       },
       "query" : "{\"match\": {\"title\": \"foo\"}}"
     } ],
+    "applications" : [ ],
     "run_as" : [ "other_user" ],
     "metadata" : {
       "version" : 1
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index aa60456d805..6e57c243630 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -133,6 +133,8 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExceptExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction;
@@ -342,6 +344,11 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
                 new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetaData.TYPE, TokenMetaData::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, TokenMetaData.TYPE, TokenMetaData::readDiffFrom),
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SECURITY, SecurityFeatureSetUsage::new),
+                // security : conditional privileges
+                new NamedWriteableRegistry.Entry(ConditionalClusterPrivilege.class,
+                    ConditionalClusterPrivileges.ManageApplicationPrivileges.WRITEABLE_NAME,
+                    ConditionalClusterPrivileges.ManageApplicationPrivileges::createFrom),
+                // security : role-mappings
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AllExpression.NAME, AllExpression::new),
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AnyExpression.NAME, AnyExpression::new),
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, FieldExpression.NAME, FieldExpression::new),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ApplicationPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ApplicationPrivilegesRequest.java
new file mode 100644
index 00000000000..3d7c7659361
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ApplicationPrivilegesRequest.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import java.util.Collection;
+
+/**
+ * Interface implemented by all Requests that manage application privileges
+ */
+public interface ApplicationPrivilegesRequest {
+
+    Collection<String> getApplicationNames();
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java
new file mode 100644
index 00000000000..a36d2fdec74
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.Action;
+
+/**
+ * Action for deleting application privileges.
+ */
+public final class DeletePrivilegesAction extends Action<DeletePrivilegesResponse> {
+
+    public static final DeletePrivilegesAction INSTANCE = new DeletePrivilegesAction();
+    public static final String NAME = "cluster:admin/xpack/security/privilege/delete";
+
+    private DeletePrivilegesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public DeletePrivilegesResponse newResponse() {
+        return new DeletePrivilegesResponse();
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequest.java
new file mode 100644
index 00000000000..d5ed78a4823
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequest.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * A request to delete an application privilege.
+ */
+public final class DeletePrivilegesRequest extends ActionRequest
+    implements ApplicationPrivilegesRequest, WriteRequest<DeletePrivilegesRequest> {
+
+    private String application;
+    private String[] privileges;
+    private RefreshPolicy refreshPolicy = RefreshPolicy.IMMEDIATE;
+
+    public DeletePrivilegesRequest() {
+        this(null, Strings.EMPTY_ARRAY);
+    }
+
+    public DeletePrivilegesRequest(String application, String[] privileges) {
+        this.application = application;
+        this.privileges = privileges;
+    }
+
+    @Override
+    public DeletePrivilegesRequest setRefreshPolicy(RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = refreshPolicy;
+        return this;
+    }
+
+    @Override
+    public RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(application)) {
+            validationException = addValidationError("application name is missing", validationException);
+        }
+        if (privileges == null || privileges.length == 0 || Arrays.stream(privileges).allMatch(Strings::isNullOrEmpty)) {
+            validationException = addValidationError("privileges are missing", validationException);
+        }
+        return validationException;
+    }
+
+    public void application(String application) {
+        this.application = application;
+    }
+
+    public String application() {
+        return application;
+    }
+
+    @Override
+    public Collection<String> getApplicationNames() {
+        return Collections.singleton(application);
+    }
+
+    public String[] privileges() {
+        return this.privileges;
+    }
+
+    public void privileges(String[] privileges) {
+        this.privileges = privileges;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        application = in.readString();
+        privileges = in.readStringArray();
+        refreshPolicy = RefreshPolicy.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(application);
+        out.writeStringArray(privileges);
+        refreshPolicy.writeTo(out);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java
new file mode 100644
index 00000000000..c1d364476ea
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.WriteRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Builder for {@link DeletePrivilegesRequest}
+ */
+public final class DeletePrivilegesRequestBuilder extends ActionRequestBuilder<DeletePrivilegesRequest, DeletePrivilegesResponse>
+        implements WriteRequestBuilder<DeletePrivilegesRequestBuilder> {
+
+    public DeletePrivilegesRequestBuilder(ElasticsearchClient client, DeletePrivilegesAction action) {
+        super(client, action, new DeletePrivilegesRequest());
+    }
+
+    public DeletePrivilegesRequestBuilder privileges(String[] privileges) {
+        request.privileges(privileges);
+        return this;
+    }
+
+    public DeletePrivilegesRequestBuilder application(String applicationName) {
+        request.application(applicationName);
+        return this;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesResponse.java
new file mode 100644
index 00000000000..18efb2ac5fa
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesResponse.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Response when deleting application privileges.
+ * Returns a collection of privileges that were successfully found and deleted.
+ */
+public final class DeletePrivilegesResponse extends ActionResponse implements ToXContentObject {
+
+    private Set<String> found;
+
+    public DeletePrivilegesResponse() {
+    }
+
+    public DeletePrivilegesResponse(Collection<String> found) {
+        this.found = Collections.unmodifiableSet(new HashSet<>(found));
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject().field("found", found).endObject();
+        return builder;
+    }
+
+    public Set<String> found() {
+        return this.found;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.found = Collections.unmodifiableSet(in.readSet(StreamInput::readString));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeCollection(found, StreamOutput::writeString);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesAction.java
new file mode 100644
index 00000000000..0b8743228c5
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesAction.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.Action;
+
+/**
+ * Action for retrieving one or more application privileges from the security index
+ */
+public final class GetPrivilegesAction extends Action<GetPrivilegesResponse> {
+
+    public static final GetPrivilegesAction INSTANCE = new GetPrivilegesAction();
+    public static final String NAME = "cluster:admin/xpack/security/privilege/get";
+
+    private GetPrivilegesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public GetPrivilegesResponse newResponse() {
+        return new GetPrivilegesResponse();
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
new file mode 100644
index 00000000000..559e0ab8d98
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request to retrieve one or more application privileges.
+ */
+public final class GetPrivilegesRequest extends ActionRequest implements ApplicationPrivilegesRequest {
+
+    @Nullable
+    private String application;
+    private String[] privileges;
+
+    public GetPrivilegesRequest() {
+        privileges = Strings.EMPTY_ARRAY;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (privileges == null) {
+            validationException = addValidationError("privileges cannot be null", validationException);
+        }
+        return validationException;
+    }
+
+    public void application(String application) {
+        this.application = application;
+    }
+
+    public String application() {
+        return this.application;
+    }
+
+    @Override
+    public Collection<String> getApplicationNames() {
+        return Collections.singleton(application);
+    }
+
+    public void privileges(String... privileges) {
+        this.privileges = privileges;
+    }
+
+    public String[] privileges() {
+        return this.privileges;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        application = in.readOptionalString();
+        privileges = in.readStringArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalString(application);
+        out.writeStringArray(privileges);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestBuilder.java
new file mode 100644
index 00000000000..305c8d1ff79
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestBuilder.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Builder for {@link GetPrivilegesRequest}
+ */
+public final class GetPrivilegesRequestBuilder extends ActionRequestBuilder<GetPrivilegesRequest, GetPrivilegesResponse> {
+
+    public GetPrivilegesRequestBuilder(ElasticsearchClient client, GetPrivilegesAction action) {
+        super(client, action, new GetPrivilegesRequest());
+    }
+
+    public GetPrivilegesRequestBuilder privileges(String... privileges) {
+        request.privileges(privileges);
+        return this;
+    }
+
+    public GetPrivilegesRequestBuilder application(String applicationName) {
+        request.application(applicationName);
+        return this;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponse.java
new file mode 100644
index 00000000000..664673aa97e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponse.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * Response containing one or more application privileges retrieved from the security index
+ */
+public final class GetPrivilegesResponse extends ActionResponse {
+
+    private ApplicationPrivilegeDescriptor[] privileges;
+
+    public GetPrivilegesResponse(ApplicationPrivilegeDescriptor... privileges) {
+        this.privileges = privileges;
+    }
+
+    public GetPrivilegesResponse(Collection<ApplicationPrivilegeDescriptor> privileges) {
+        this(privileges.toArray(new ApplicationPrivilegeDescriptor[privileges.size()]));
+    }
+
+    public ApplicationPrivilegeDescriptor[] privileges() {
+        return privileges;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.privileges = in.readArray(ApplicationPrivilegeDescriptor::new, ApplicationPrivilegeDescriptor[]::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeArray(privileges);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesAction.java
new file mode 100644
index 00000000000..3743bec144f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesAction.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.Action;
+
+/**
+ * Action for putting (adding/updating) one or more application privileges.
+ */
+public final class PutPrivilegesAction extends Action<PutPrivilegesResponse> {
+
+    public static final PutPrivilegesAction INSTANCE = new PutPrivilegesAction();
+    public static final String NAME = "cluster:admin/xpack/security/privilege/put";
+
+    private PutPrivilegesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public PutPrivilegesResponse newResponse() {
+        return new PutPrivilegesResponse();
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequest.java
new file mode 100644
index 00000000000..beba805f6df
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequest.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.support.MetadataUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request object to put a one or more application privileges.
+ */
+public final class PutPrivilegesRequest extends ActionRequest implements ApplicationPrivilegesRequest, WriteRequest<PutPrivilegesRequest> {
+
+    private List<ApplicationPrivilegeDescriptor> privileges;
+    private RefreshPolicy refreshPolicy = RefreshPolicy.IMMEDIATE;
+
+    public PutPrivilegesRequest() {
+        privileges = Collections.emptyList();
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        for (ApplicationPrivilegeDescriptor privilege : privileges) {
+            try {
+                ApplicationPrivilege.validateApplicationName(privilege.getApplication());
+            } catch (IllegalArgumentException e) {
+                validationException = addValidationError(e.getMessage(), validationException);
+            }
+            try {
+                ApplicationPrivilege.validatePrivilegeName(privilege.getName());
+            } catch (IllegalArgumentException e) {
+                validationException = addValidationError(e.getMessage(), validationException);
+            }
+            if (privilege.getActions().isEmpty()) {
+                validationException = addValidationError("Application privileges must have at least one action", validationException);
+            }
+            for (String action : privilege.getActions()) {
+                if (action.indexOf('/') == -1 && action.indexOf('*') == -1 && action.indexOf(':') == -1) {
+                    validationException = addValidationError("action [" + action + "] must contain one of [ '/' , '*' , ':' ]",
+                        validationException);
+                }
+                try {
+                    ApplicationPrivilege.validatePrivilegeOrActionName(action);
+                } catch (IllegalArgumentException e) {
+                    validationException = addValidationError(e.getMessage(), validationException);
+                }
+            }
+            if (MetadataUtils.containsReservedMetadata(privilege.getMetadata())) {
+                validationException = addValidationError("metadata keys may not start with [" + MetadataUtils.RESERVED_PREFIX
+                    + "] (in privilege " + privilege.getApplication() + ' ' + privilege.getName() + ")", validationException);
+            }
+        }
+        return validationException;
+    }
+
+    /**
+     * Should this request trigger a refresh ({@linkplain RefreshPolicy#IMMEDIATE}, the default), wait for a refresh (
+     * {@linkplain RefreshPolicy#WAIT_UNTIL}), or proceed ignore refreshes entirely ({@linkplain RefreshPolicy#NONE}).
+     */
+    @Override
+    public RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public PutPrivilegesRequest setRefreshPolicy(RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = refreshPolicy;
+        return this;
+    }
+
+    public List<ApplicationPrivilegeDescriptor> getPrivileges() {
+        return privileges;
+    }
+
+    public void setPrivileges(Collection<ApplicationPrivilegeDescriptor> privileges) {
+        this.privileges = Collections.unmodifiableList(new ArrayList<>(privileges));
+    }
+
+    @Override
+    public Collection<String> getApplicationNames() {
+        return Collections.unmodifiableSet(privileges.stream()
+            .map(ApplicationPrivilegeDescriptor::getApplication)
+            .collect(Collectors.toSet()));
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{[" + privileges.stream().map(Strings::toString).collect(Collectors.joining(","))
+            + "];" + refreshPolicy + "}";
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        privileges = Collections.unmodifiableList(in.readList(ApplicationPrivilegeDescriptor::new));
+        refreshPolicy = RefreshPolicy.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeList(privileges);
+        refreshPolicy.writeTo(out);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java
new file mode 100644
index 00000000000..b8c2685d28a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.WriteRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Request builder for {@link PutPrivilegesRequest}
+ */
+public final class PutPrivilegesRequestBuilder extends ActionRequestBuilder<PutPrivilegesRequest, PutPrivilegesResponse>
+    implements WriteRequestBuilder<PutPrivilegesRequestBuilder> {
+
+    public PutPrivilegesRequestBuilder(ElasticsearchClient client, PutPrivilegesAction action) {
+        super(client, action, new PutPrivilegesRequest());
+    }
+
+    /**
+     * Populate the put privileges request using the given source, application name and privilege name
+     * The source must contain a single privilege object which matches the application and privilege names.
+     */
+    public PutPrivilegesRequestBuilder source(String applicationName, String expectedName,
+                                              BytesReference source, XContentType xContentType)
+        throws IOException {
+        Objects.requireNonNull(xContentType);
+        // EMPTY is ok here because we never call namedObject
+        try (InputStream stream = source.streamInput();
+             XContentParser parser = xContentType.xContent()
+                 .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
+            XContentParser.Token token = parser.currentToken();
+            if (token == null) {
+                token = parser.nextToken();
+            }
+            if (token == XContentParser.Token.START_OBJECT) {
+                final ApplicationPrivilegeDescriptor privilege = parsePrivilege(parser, applicationName, expectedName);
+                this.request.setPrivileges(Collections.singleton(privilege));
+            } else {
+                throw new ElasticsearchParseException("expected an object but found {} instead", token);
+            }
+        }
+        return this;
+    }
+
+    ApplicationPrivilegeDescriptor parsePrivilege(XContentParser parser, String applicationName, String privilegeName) throws IOException {
+        ApplicationPrivilegeDescriptor privilege = ApplicationPrivilegeDescriptor.parse(parser, applicationName, privilegeName, false);
+        checkPrivilegeName(privilege, applicationName, privilegeName);
+        return privilege;
+    }
+
+    /**
+     * Populate the put privileges request using the given source, application name and privilege name
+     * The source must contain a top-level object, keyed by application name.
+     * The value for each application-name, is an object keyed by privilege name.
+     * The value for each privilege-name is a privilege object which much match the application and privilege names in which it is nested.
+     */
+    public PutPrivilegesRequestBuilder source(BytesReference source, XContentType xContentType)
+        throws IOException {
+        Objects.requireNonNull(xContentType);
+        // EMPTY is ok here because we never call namedObject
+        try (InputStream stream = source.streamInput();
+             XContentParser parser = xContentType.xContent()
+                 .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
+            XContentParser.Token token = parser.currentToken();
+            if (token == null) {
+                token = parser.nextToken();
+            }
+            if (token != XContentParser.Token.START_OBJECT) {
+                throw new ElasticsearchParseException("expected object but found {} instead", token);
+            }
+
+            List<ApplicationPrivilegeDescriptor> privileges = new ArrayList<>();
+            while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                token = parser.currentToken();
+                assert token == XContentParser.Token.FIELD_NAME : "Invalid token " + token;
+                final String applicationName = parser.currentName();
+
+                token = parser.nextToken();
+                if (token != XContentParser.Token.START_OBJECT) {
+                    throw new ElasticsearchParseException("expected the value for {} to be an object, but found {} instead",
+                        applicationName, token);
+                }
+
+                while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                    token = parser.currentToken();
+                    assert (token == XContentParser.Token.FIELD_NAME);
+                    final String privilegeName = parser.currentName();
+
+                    token = parser.nextToken();
+                    if (token != XContentParser.Token.START_OBJECT) {
+                        throw new ElasticsearchParseException("expected the value for {} to be an object, but found {} instead",
+                            applicationName, token);
+                    }
+                    privileges.add(parsePrivilege(parser, applicationName, privilegeName));
+                }
+            }
+            request.setPrivileges(privileges);
+        }
+        return this;
+    }
+
+    private void checkPrivilegeName(ApplicationPrivilegeDescriptor privilege, String applicationName, String providedName) {
+        final String privilegeName = privilege.getName();
+        if (Strings.isNullOrEmpty(applicationName) == false && applicationName.equals(privilege.getApplication()) == false) {
+            throw new IllegalArgumentException("privilege application [" + privilege.getApplication()
+                + "] in source does not match the provided application [" + applicationName + "]");
+        }
+        if (Strings.isNullOrEmpty(providedName) == false && providedName.equals(privilegeName) == false) {
+            throw new IllegalArgumentException("privilege name [" + privilegeName
+                + "] in source does not match the provided name [" + providedName + "]");
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesResponse.java
new file mode 100644
index 00000000000..6d4a3f1ad44
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesResponse.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Response when adding one or more application privileges to the security index.
+ * Returns a collection of the privileges that were created (by implication, any other privileges were updated).
+ */
+public final class PutPrivilegesResponse extends ActionResponse implements ToXContentObject {
+
+    private Map<String, List<String>> created;
+
+    PutPrivilegesResponse() {
+        this(Collections.emptyMap());
+    }
+
+    public PutPrivilegesResponse(Map<String, List<String>> created) {
+        this.created = Collections.unmodifiableMap(created);
+    }
+
+    /**
+     * Get a list of privileges that were created (as opposed to updated)
+     * @return A map from <em>Application Name</em> to a {@code List} of <em>privilege names</em>
+     */
+    public Map<String, List<String>> created() {
+        return created;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject().field("created", created).endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeMap(created, StreamOutput::writeString, StreamOutput::writeStringList);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.created = Collections.unmodifiableMap(in.readMap(StreamInput::readString, si -> si.readList(StreamInput::readString)));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
index d0f3423fdcf..96c9c817182 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.role;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.WriteRequest;
@@ -14,11 +15,15 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -31,11 +36,13 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
 
     private String name;
     private String[] clusterPrivileges = Strings.EMPTY_ARRAY;
+    private ConditionalClusterPrivilege[] conditionalClusterPrivileges = ConditionalClusterPrivileges.EMPTY_ARRAY;
     private List<RoleDescriptor.IndicesPrivileges> indicesPrivileges = new ArrayList<>();
+    private List<RoleDescriptor.ApplicationResourcePrivileges> applicationPrivileges = new ArrayList<>();
     private String[] runAs = Strings.EMPTY_ARRAY;
     private RefreshPolicy refreshPolicy = RefreshPolicy.IMMEDIATE;
     private Map<String, Object> metadata;
-    
+
     public PutRoleRequest() {
     }
 
@@ -45,9 +52,25 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
         if (name == null) {
             validationException = addValidationError("role name is missing", validationException);
         }
+        if(applicationPrivileges != null) {
+            for (RoleDescriptor.ApplicationResourcePrivileges privilege : applicationPrivileges) {
+                try {
+                    ApplicationPrivilege.validateApplicationNameOrWildcard(privilege.getApplication());
+                } catch (IllegalArgumentException e) {
+                    validationException = addValidationError(e.getMessage(), validationException);
+                }
+                for (String name : privilege.getPrivileges()) {
+                    try {
+                        ApplicationPrivilege.validatePrivilegeOrActionName(name);
+                    } catch (IllegalArgumentException e) {
+                        validationException = addValidationError(e.getMessage(), validationException);
+                    }
+                }
+            }
+        }
         if (metadata != null && MetadataUtils.containsReservedMetadata(metadata)) {
             validationException =
-                    addValidationError("metadata keys may not start with [" + MetadataUtils.RESERVED_PREFIX + "]", validationException);
+                addValidationError("metadata keys may not start with [" + MetadataUtils.RESERVED_PREFIX + "]", validationException);
         }
         return validationException;
     }
@@ -60,6 +83,10 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
         this.clusterPrivileges = clusterPrivileges;
     }
 
+    void conditionalCluster(ConditionalClusterPrivilege... conditionalClusterPrivileges) {
+        this.conditionalClusterPrivileges = conditionalClusterPrivileges;
+    }
+
     void addIndex(RoleDescriptor.IndicesPrivileges... privileges) {
         this.indicesPrivileges.addAll(Arrays.asList(privileges));
     }
@@ -75,6 +102,10 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
                 .build());
     }
 
+    void addApplicationPrivileges(RoleDescriptor.ApplicationResourcePrivileges... privileges) {
+        this.applicationPrivileges.addAll(Arrays.asList(privileges));
+    }
+
     public void runAs(String... usernames) {
         this.runAs = usernames;
     }
@@ -110,6 +141,14 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
         return indicesPrivileges.toArray(new RoleDescriptor.IndicesPrivileges[indicesPrivileges.size()]);
     }
 
+    public List<RoleDescriptor.ApplicationResourcePrivileges> applicationPrivileges() {
+        return Collections.unmodifiableList(applicationPrivileges);
+    }
+
+    public ConditionalClusterPrivilege[] conditionalClusterPrivileges() {
+        return conditionalClusterPrivileges;
+    }
+
     public String[] runAs() {
         return runAs;
     }
@@ -128,6 +167,10 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
         for (int i = 0; i < indicesSize; i++) {
             indicesPrivileges.add(RoleDescriptor.IndicesPrivileges.createFrom(in));
         }
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            applicationPrivileges = in.readList(RoleDescriptor.ApplicationResourcePrivileges::createFrom);
+            conditionalClusterPrivileges = ConditionalClusterPrivileges.readArray(in);
+        }
         runAs = in.readStringArray();
         refreshPolicy = RefreshPolicy.readFrom(in);
         metadata = in.readMap();
@@ -142,6 +185,10 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
         for (RoleDescriptor.IndicesPrivileges index : indicesPrivileges) {
             index.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeStreamableList(applicationPrivileges);
+            ConditionalClusterPrivileges.writeArray(out, this.conditionalClusterPrivileges);
+        }
         out.writeStringArray(runAs);
         refreshPolicy.writeTo(out);
         out.writeMap(metadata);
@@ -151,7 +198,11 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
         return new RoleDescriptor(name,
                 clusterPrivileges,
                 indicesPrivileges.toArray(new RoleDescriptor.IndicesPrivileges[indicesPrivileges.size()]),
+                applicationPrivileges.toArray(new RoleDescriptor.ApplicationResourcePrivileges[applicationPrivileges.size()]),
+                conditionalClusterPrivileges,
                 runAs,
-                metadata);
+                metadata,
+                Collections.emptyMap());
     }
-}
\ No newline at end of file
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilder.java
index 25d443eda3f..670deb2216b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilder.java
@@ -40,7 +40,9 @@ public class PutRoleRequestBuilder extends ActionRequestBuilder<PutRoleRequest,
         assert name.equals(descriptor.getName());
         request.name(name);
         request.cluster(descriptor.getClusterPrivileges());
+        request.conditionalCluster(descriptor.getConditionalClusterPrivileges());
         request.addIndex(descriptor.getIndicesPrivileges());
+        request.addApplicationPrivileges(descriptor.getApplicationPrivileges());
         request.runAs(descriptor.getRunAs());
         request.metadata(descriptor.getMetadata());
         return this;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequest.java
index 101ae00d635..dc43db0115e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequest.java
@@ -5,11 +5,14 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 
 import java.io.IOException;
 
@@ -23,6 +26,7 @@ public class HasPrivilegesRequest extends ActionRequest implements UserRequest {
     private String username;
     private String[] clusterPrivileges;
     private RoleDescriptor.IndicesPrivileges[] indexPrivileges;
+    private ApplicationResourcePrivileges[] applicationPrivileges;
 
     @Override
     public ActionRequestValidationException validate() {
@@ -33,9 +37,21 @@ public class HasPrivilegesRequest extends ActionRequest implements UserRequest {
         if (indexPrivileges == null) {
             validationException = addValidationError("indexPrivileges must not be null", validationException);
         }
-        if (clusterPrivileges != null && clusterPrivileges.length == 0 && indexPrivileges != null && indexPrivileges.length == 0) {
-            validationException = addValidationError("clusterPrivileges and indexPrivileges cannot both be empty",
-                    validationException);
+        if (applicationPrivileges == null) {
+            validationException = addValidationError("applicationPrivileges must not be null", validationException);
+        } else {
+            for (ApplicationResourcePrivileges applicationPrivilege : applicationPrivileges) {
+                try {
+                    ApplicationPrivilege.validateApplicationName(applicationPrivilege.getApplication());
+                } catch (IllegalArgumentException e) {
+                    validationException = addValidationError(e.getMessage(), validationException);
+                }
+            }
+        }
+        if (clusterPrivileges != null && clusterPrivileges.length == 0
+            && indexPrivileges != null && indexPrivileges.length == 0
+            && applicationPrivileges != null && applicationPrivileges.length == 0) {
+            validationException = addValidationError("must specify at least one privilege", validationException);
         }
         return validationException;
     }
@@ -67,6 +83,10 @@ public class HasPrivilegesRequest extends ActionRequest implements UserRequest {
         return clusterPrivileges;
     }
 
+    public ApplicationResourcePrivileges[] applicationPrivileges() {
+        return applicationPrivileges;
+    }
+
     public void indexPrivileges(RoleDescriptor.IndicesPrivileges... privileges) {
         this.indexPrivileges = privileges;
     }
@@ -75,6 +95,10 @@ public class HasPrivilegesRequest extends ActionRequest implements UserRequest {
         this.clusterPrivileges = privileges;
     }
 
+    public void applicationPrivileges(ApplicationResourcePrivileges... appPrivileges) {
+        this.applicationPrivileges = appPrivileges;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -85,6 +109,9 @@ public class HasPrivilegesRequest extends ActionRequest implements UserRequest {
         for (int i = 0; i < indexSize; i++) {
             indexPrivileges[i] = RoleDescriptor.IndicesPrivileges.createFrom(in);
         }
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            applicationPrivileges = in.readArray(ApplicationResourcePrivileges::createFrom, ApplicationResourcePrivileges[]::new);
+        }
     }
 
     @Override
@@ -96,6 +123,9 @@ public class HasPrivilegesRequest extends ActionRequest implements UserRequest {
         for (RoleDescriptor.IndicesPrivileges priv : indexPrivileges) {
             priv.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeArray(ApplicationResourcePrivileges::write, applicationPrivileges);
+        }
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestBuilder.java
index 4504a95962c..bf705da1a1f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestBuilder.java
@@ -39,6 +39,7 @@ public class HasPrivilegesRequestBuilder
         request.username(username);
         request.indexPrivileges(role.getIndicesPrivileges());
         request.clusterPrivileges(role.getClusterPrivileges());
+        request.applicationPrivileges(role.getApplicationPrivileges());
         return this;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java
index dcc34d75ddb..b0711fc1bc1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java
@@ -5,6 +5,11 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -14,27 +19,27 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-
 /**
  * Response for a {@link HasPrivilegesRequest}
  */
 public class HasPrivilegesResponse extends ActionResponse {
     private boolean completeMatch;
     private Map<String, Boolean> cluster;
-    private List<IndexPrivileges> index;
+    private List<ResourcePrivileges> index;
+    private Map<String, List<ResourcePrivileges>> application;
 
     public HasPrivilegesResponse() {
-        this(true, Collections.emptyMap(), Collections.emptyList());
+        this(true, Collections.emptyMap(), Collections.emptyList(), Collections.emptyMap());
     }
 
-    public HasPrivilegesResponse(boolean completeMatch, Map<String, Boolean> cluster, Collection<IndexPrivileges> index) {
+    public HasPrivilegesResponse(boolean completeMatch, Map<String, Boolean> cluster, Collection<ResourcePrivileges> index,
+                                 Map<String, Collection<ResourcePrivileges>> application) {
         super();
         this.completeMatch = completeMatch;
         this.cluster = new HashMap<>(cluster);
         this.index = new ArrayList<>(index);
+        this.application = new HashMap<>();
+        application.forEach((key, val) -> this.application.put(key, Collections.unmodifiableList(new ArrayList<>(val))));
     }
 
     public boolean isCompleteMatch() {
@@ -45,44 +50,67 @@ public class HasPrivilegesResponse extends ActionResponse {
         return Collections.unmodifiableMap(cluster);
     }
 
-    public List<IndexPrivileges> getIndexPrivileges() {
+    public List<ResourcePrivileges> getIndexPrivileges() {
         return Collections.unmodifiableList(index);
     }
 
+    /**
+     * Retrieves the results from checking application privileges,
+     * @return A {@code Map} keyed by application-name
+     */
+    public Map<String, List<ResourcePrivileges>> getApplicationPrivileges() {
+        return Collections.unmodifiableMap(application);
+    }
+
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         completeMatch = in.readBoolean();
-        int count = in.readVInt();
-        index = new ArrayList<>(count);
+        index = readResourcePrivileges(in);
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            application = in.readMap(StreamInput::readString, HasPrivilegesResponse::readResourcePrivileges);
+        }
+    }
+
+    private static List<ResourcePrivileges> readResourcePrivileges(StreamInput in) throws IOException {
+        final int count = in.readVInt();
+        final List<ResourcePrivileges> list = new ArrayList<>(count);
         for (int i = 0; i < count; i++) {
             final String index = in.readString();
             final Map<String, Boolean> privileges = in.readMap(StreamInput::readString, StreamInput::readBoolean);
-            this.index.add(new IndexPrivileges(index, privileges));
+            list.add(new ResourcePrivileges(index, privileges));
         }
+        return list;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeBoolean(completeMatch);
-        out.writeVInt(index.size());
-        for (IndexPrivileges index : index) {
-            out.writeString(index.index);
-            out.writeMap(index.privileges, StreamOutput::writeString, StreamOutput::writeBoolean);
+        writeResourcePrivileges(out, index);
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeMap(application, StreamOutput::writeString, HasPrivilegesResponse::writeResourcePrivileges);
         }
     }
 
-    public static class IndexPrivileges {
-        private final String index;
+    private static void writeResourcePrivileges(StreamOutput out, List<ResourcePrivileges> privileges) throws IOException {
+        out.writeVInt(privileges.size());
+        for (ResourcePrivileges priv : privileges) {
+            out.writeString(priv.resource);
+            out.writeMap(priv.privileges, StreamOutput::writeString, StreamOutput::writeBoolean);
+        }
+    }
+
+    public static class ResourcePrivileges {
+        private final String resource;
         private final Map<String, Boolean> privileges;
 
-        public IndexPrivileges(String index, Map<String, Boolean> privileges) {
-            this.index = Objects.requireNonNull(index);
+        public ResourcePrivileges(String resource, Map<String, Boolean> privileges) {
+            this.resource = Objects.requireNonNull(resource);
             this.privileges = Collections.unmodifiableMap(privileges);
         }
 
-        public String getIndex() {
-            return index;
+        public String getResource() {
+            return resource;
         }
 
         public Map<String, Boolean> getPrivileges() {
@@ -92,14 +120,14 @@ public class HasPrivilegesResponse extends ActionResponse {
         @Override
         public String toString() {
             return getClass().getSimpleName() + "{" +
-                    "index='" + index + '\'' +
+                    "resource='" + resource + '\'' +
                     ", privileges=" + privileges +
                     '}';
         }
 
         @Override
         public int hashCode() {
-            int result = index.hashCode();
+            int result = resource.hashCode();
             result = 31 * result + privileges.hashCode();
             return result;
         }
@@ -113,8 +141,8 @@ public class HasPrivilegesResponse extends ActionResponse {
                 return false;
             }
 
-            final IndexPrivileges other = (IndexPrivileges) o;
-            return this.index.equals(other.index) && this.privileges.equals(other.privileges);
+            final ResourcePrivileges other = (ResourcePrivileges) o;
+            return this.resource.equals(other.resource) && this.privileges.equals(other.privileges);
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
index 65bde9bd1df..42bd771103f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
@@ -18,11 +18,14 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 import org.elasticsearch.xpack.core.security.support.Validation;
 import org.elasticsearch.xpack.core.security.xcontent.XContentUtils;
@@ -31,9 +34,11 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * A holder for a Role that contains user-readable information about the Role
@@ -45,7 +50,9 @@ public class RoleDescriptor implements ToXContentObject {
 
     private final String name;
     private final String[] clusterPrivileges;
+    private final ConditionalClusterPrivilege[] conditionalClusterPrivileges;
     private final IndicesPrivileges[] indicesPrivileges;
+    private final ApplicationResourcePrivileges[] applicationPrivileges;
     private final String[] runAs;
     private final Map<String, Object> metadata;
     private final Map<String, Object> transientMetadata;
@@ -57,6 +64,11 @@ public class RoleDescriptor implements ToXContentObject {
         this(name, clusterPrivileges, indicesPrivileges, runAs, null);
     }
 
+    /**
+     * @deprecated Use {@link #RoleDescriptor(String, String[], IndicesPrivileges[], ApplicationResourcePrivileges[],
+     * ConditionalClusterPrivilege[], String[], Map, Map)}
+     */
+    @Deprecated
     public RoleDescriptor(String name,
                           @Nullable String[] clusterPrivileges,
                           @Nullable IndicesPrivileges[] indicesPrivileges,
@@ -65,16 +77,34 @@ public class RoleDescriptor implements ToXContentObject {
         this(name, clusterPrivileges, indicesPrivileges, runAs, metadata, null);
     }
 
-
+    /**
+     * @deprecated Use {@link #RoleDescriptor(String, String[], IndicesPrivileges[], ApplicationResourcePrivileges[],
+     * ConditionalClusterPrivilege[], String[], Map, Map)}
+     */
+    @Deprecated
     public RoleDescriptor(String name,
                           @Nullable String[] clusterPrivileges,
                           @Nullable IndicesPrivileges[] indicesPrivileges,
                           @Nullable String[] runAs,
                           @Nullable Map<String, Object> metadata,
                           @Nullable Map<String, Object> transientMetadata) {
+        this(name, clusterPrivileges, indicesPrivileges, null, null, runAs, metadata, transientMetadata);
+    }
+
+    public RoleDescriptor(String name,
+                          @Nullable String[] clusterPrivileges,
+                          @Nullable IndicesPrivileges[] indicesPrivileges,
+                          @Nullable ApplicationResourcePrivileges[] applicationPrivileges,
+                          @Nullable ConditionalClusterPrivilege[] conditionalClusterPrivileges,
+                          @Nullable String[] runAs,
+                          @Nullable Map<String, Object> metadata,
+                          @Nullable Map<String, Object> transientMetadata) {
         this.name = name;
         this.clusterPrivileges = clusterPrivileges != null ? clusterPrivileges : Strings.EMPTY_ARRAY;
+        this.conditionalClusterPrivileges = conditionalClusterPrivileges != null
+            ? conditionalClusterPrivileges : ConditionalClusterPrivileges.EMPTY_ARRAY;
         this.indicesPrivileges = indicesPrivileges != null ? indicesPrivileges : IndicesPrivileges.NONE;
+        this.applicationPrivileges = applicationPrivileges != null ? applicationPrivileges : ApplicationResourcePrivileges.NONE;
         this.runAs = runAs != null ? runAs : Strings.EMPTY_ARRAY;
         this.metadata = metadata != null ? Collections.unmodifiableMap(metadata) : Collections.emptyMap();
         this.transientMetadata = transientMetadata != null ? Collections.unmodifiableMap(transientMetadata) :
@@ -89,10 +119,18 @@ public class RoleDescriptor implements ToXContentObject {
         return this.clusterPrivileges;
     }
 
+    public ConditionalClusterPrivilege[] getConditionalClusterPrivileges() {
+        return this.conditionalClusterPrivileges;
+    }
+
     public IndicesPrivileges[] getIndicesPrivileges() {
         return this.indicesPrivileges;
     }
 
+    public ApplicationResourcePrivileges[] getApplicationPrivileges() {
+        return this.applicationPrivileges;
+    }
+
     public String[] getRunAs() {
         return this.runAs;
     }
@@ -114,10 +152,15 @@ public class RoleDescriptor implements ToXContentObject {
         StringBuilder sb = new StringBuilder("Role[");
         sb.append("name=").append(name);
         sb.append(", cluster=[").append(Strings.arrayToCommaDelimitedString(clusterPrivileges));
+        sb.append("], global=[").append(Strings.arrayToCommaDelimitedString(conditionalClusterPrivileges));
         sb.append("], indicesPrivileges=[");
         for (IndicesPrivileges group : indicesPrivileges) {
             sb.append(group.toString()).append(",");
         }
+        sb.append("], applicationPrivileges=[");
+        for (ApplicationResourcePrivileges privilege : applicationPrivileges) {
+            sb.append(privilege.toString()).append(",");
+        }
         sb.append("], runAs=[").append(Strings.arrayToCommaDelimitedString(runAs));
         sb.append("], metadata=[");
         MetadataUtils.writeValue(sb, metadata);
@@ -134,7 +177,9 @@ public class RoleDescriptor implements ToXContentObject {
 
         if (!name.equals(that.name)) return false;
         if (!Arrays.equals(clusterPrivileges, that.clusterPrivileges)) return false;
+        if (!Arrays.equals(conditionalClusterPrivileges, that.conditionalClusterPrivileges)) return false;
         if (!Arrays.equals(indicesPrivileges, that.indicesPrivileges)) return false;
+        if (!Arrays.equals(applicationPrivileges, that.applicationPrivileges)) return false;
         if (!metadata.equals(that.getMetadata())) return false;
         return Arrays.equals(runAs, that.runAs);
     }
@@ -143,7 +188,9 @@ public class RoleDescriptor implements ToXContentObject {
     public int hashCode() {
         int result = name.hashCode();
         result = 31 * result + Arrays.hashCode(clusterPrivileges);
+        result = 31 * result + Arrays.hashCode(conditionalClusterPrivileges);
         result = 31 * result + Arrays.hashCode(indicesPrivileges);
+        result = 31 * result + Arrays.hashCode(applicationPrivileges);
         result = 31 * result + Arrays.hashCode(runAs);
         result = 31 * result + metadata.hashCode();
         return result;
@@ -157,8 +204,8 @@ public class RoleDescriptor implements ToXContentObject {
     /**
      * Generates x-content for this {@link RoleDescriptor} instance.
      *
-     * @param builder the x-content builder
-     * @param params the parameters for x-content generation directives
+     * @param builder     the x-content builder
+     * @param params      the parameters for x-content generation directives
      * @param docCreation {@code true} if the x-content is being generated for creating a document
      *                    in the security index, {@code false} if the x-content being generated
      *                    is for API display purposes
@@ -168,7 +215,12 @@ public class RoleDescriptor implements ToXContentObject {
     public XContentBuilder toXContent(XContentBuilder builder, Params params, boolean docCreation) throws IOException {
         builder.startObject();
         builder.array(Fields.CLUSTER.getPreferredName(), clusterPrivileges);
+        if (conditionalClusterPrivileges.length != 0) {
+            builder.field(Fields.GLOBAL.getPreferredName());
+            ConditionalClusterPrivileges.toXContent(builder, params, Arrays.asList(conditionalClusterPrivileges));
+        }
         builder.array(Fields.INDICES.getPreferredName(), (Object[]) indicesPrivileges);
+        builder.array(Fields.APPLICATIONS.getPreferredName(), (Object[]) applicationPrivileges);
         if (runAs != null) {
             builder.array(Fields.RUN_AS.getPreferredName(), runAs);
         }
@@ -198,7 +250,19 @@ public class RoleDescriptor implements ToXContentObject {
         } else {
             transientMetadata = Collections.emptyMap();
         }
-        return new RoleDescriptor(name, clusterPrivileges, indicesPrivileges, runAs, metadata, transientMetadata);
+
+        final ApplicationResourcePrivileges[] applicationPrivileges;
+        final ConditionalClusterPrivilege[] conditionalClusterPrivileges;
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            applicationPrivileges = in.readArray(ApplicationResourcePrivileges::createFrom, ApplicationResourcePrivileges[]::new);
+            conditionalClusterPrivileges = ConditionalClusterPrivileges.readArray(in);
+        } else {
+            applicationPrivileges = ApplicationResourcePrivileges.NONE;
+            conditionalClusterPrivileges = ConditionalClusterPrivileges.EMPTY_ARRAY;
+        }
+
+        return new RoleDescriptor(name, clusterPrivileges, indicesPrivileges, applicationPrivileges, conditionalClusterPrivileges,
+            runAs, metadata, transientMetadata);
     }
 
     public static void writeTo(RoleDescriptor descriptor, StreamOutput out) throws IOException {
@@ -213,6 +277,10 @@ public class RoleDescriptor implements ToXContentObject {
         if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
             out.writeMap(descriptor.transientMetadata);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeArray(ApplicationResourcePrivileges::write, descriptor.applicationPrivileges);
+            ConditionalClusterPrivileges.writeArray(out, descriptor.getConditionalClusterPrivileges());
+        }
     }
 
     public static RoleDescriptor parse(String name, BytesReference source, boolean allow2xFormat, XContentType xContentType)
@@ -221,7 +289,7 @@ public class RoleDescriptor implements ToXContentObject {
         // EMPTY is safe here because we never use namedObject
         try (InputStream stream = source.streamInput();
              XContentParser parser = xContentType.xContent()
-                .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
+                     .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
             return parse(name, parser, allow2xFormat);
         }
     }
@@ -243,6 +311,8 @@ public class RoleDescriptor implements ToXContentObject {
         String currentFieldName = null;
         IndicesPrivileges[] indicesPrivileges = null;
         String[] clusterPrivileges = null;
+        List<ConditionalClusterPrivilege> conditionalClusterPrivileges = Collections.emptyList();
+        ApplicationResourcePrivileges[] applicationPrivileges = null;
         String[] runAsUsers = null;
         Map<String, Object> metadata = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -255,6 +325,11 @@ public class RoleDescriptor implements ToXContentObject {
                 runAsUsers = readStringArray(name, parser, true);
             } else if (Fields.CLUSTER.match(currentFieldName, parser.getDeprecationHandler())) {
                 clusterPrivileges = readStringArray(name, parser, true);
+            } else if (Fields.APPLICATIONS.match(currentFieldName, parser.getDeprecationHandler())
+                    || Fields.APPLICATION.match(currentFieldName, parser.getDeprecationHandler())) {
+                applicationPrivileges = parseApplicationPrivileges(name, parser);
+            } else if (Fields.GLOBAL.match(currentFieldName, parser.getDeprecationHandler())) {
+                conditionalClusterPrivileges = ConditionalClusterPrivileges.parse(parser);
             } else if (Fields.METADATA.match(currentFieldName, parser.getDeprecationHandler())) {
                 if (token != XContentParser.Token.START_OBJECT) {
                     throw new ElasticsearchParseException(
@@ -266,8 +341,7 @@ public class RoleDescriptor implements ToXContentObject {
                     // consume object but just drop
                     parser.map();
                 } else {
-                    throw new ElasticsearchParseException("expected field [{}] to be an object, but found [{}] instead",
-                            currentFieldName, token);
+                    throw new ElasticsearchParseException("failed to parse role [{}]. unexpected field [{}]", name, currentFieldName);
                 }
             } else if (Fields.TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
                 // don't need it
@@ -275,7 +349,9 @@ public class RoleDescriptor implements ToXContentObject {
                 throw new ElasticsearchParseException("failed to parse role [{}]. unexpected field [{}]", name, currentFieldName);
             }
         }
-        return new RoleDescriptor(name, clusterPrivileges, indicesPrivileges, runAsUsers, metadata);
+        return new RoleDescriptor(name, clusterPrivileges, indicesPrivileges, applicationPrivileges,
+            conditionalClusterPrivileges.toArray(new ConditionalClusterPrivilege[conditionalClusterPrivileges.size()]), runAsUsers,
+            metadata, null);
     }
 
     private static String[] readStringArray(String roleName, XContentParser parser, boolean allowNull) throws IOException {
@@ -291,7 +367,7 @@ public class RoleDescriptor implements ToXContentObject {
             throws IOException {
         try (InputStream stream = source.streamInput();
              XContentParser parser = xContentType.xContent()
-                .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
+                     .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) {
             // advance to the START_OBJECT token
             XContentParser.Token token = parser.nextToken();
             if (token != XContentParser.Token.START_OBJECT) {
@@ -301,6 +377,7 @@ public class RoleDescriptor implements ToXContentObject {
             String currentFieldName = null;
             IndicesPrivileges[] indexPrivileges = null;
             String[] clusterPrivileges = null;
+            ApplicationResourcePrivileges[] applicationPrivileges = null;
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
@@ -308,14 +385,17 @@ public class RoleDescriptor implements ToXContentObject {
                     indexPrivileges = parseIndices(description, parser, false);
                 } else if (Fields.CLUSTER.match(currentFieldName, parser.getDeprecationHandler())) {
                     clusterPrivileges = readStringArray(description, parser, true);
+                } else if (Fields.APPLICATIONS.match(currentFieldName, parser.getDeprecationHandler())
+                        || Fields.APPLICATION.match(currentFieldName, parser.getDeprecationHandler())) {
+                    applicationPrivileges = parseApplicationPrivileges(description, parser);
                 } else {
                     throw new ElasticsearchParseException("failed to parse privileges check [{}]. unexpected field [{}]",
                             description, currentFieldName);
                 }
             }
-            if (indexPrivileges == null && clusterPrivileges == null) {
-                throw new ElasticsearchParseException("failed to parse privileges check [{}]. fields [{}] and [{}] are both missing",
-                        description, Fields.INDEX, Fields.CLUSTER);
+            if (indexPrivileges == null && clusterPrivileges == null && applicationPrivileges == null) {
+                throw new ElasticsearchParseException("failed to parse privileges check [{}]. All privilege fields [{},{},{}] are missing",
+                        description, Fields.CLUSTER, Fields.INDEX, Fields.APPLICATIONS);
             }
             if (indexPrivileges != null) {
                 if (Arrays.stream(indexPrivileges).anyMatch(IndicesPrivileges::isUsingFieldLevelSecurity)) {
@@ -326,7 +406,7 @@ public class RoleDescriptor implements ToXContentObject {
                     throw new ElasticsearchParseException("Field [{}] is not supported in a has_privileges request", Fields.QUERY);
                 }
             }
-            return new RoleDescriptor(description, clusterPrivileges, indexPrivileges, null);
+            return new RoleDescriptor(description, clusterPrivileges, indexPrivileges, applicationPrivileges, null, null, null, null);
         }
     }
 
@@ -361,7 +441,7 @@ public class RoleDescriptor implements ToXContentObject {
                 currentFieldName = parser.currentName();
             } else if (Fields.NAMES.match(currentFieldName, parser.getDeprecationHandler())) {
                 if (token == XContentParser.Token.VALUE_STRING) {
-                    names = new String[] { parser.text() };
+                    names = new String[]{parser.text()};
                 } else if (token == XContentParser.Token.START_ARRAY) {
                     names = readStringArray(roleName, parser, false);
                     if (names.length == 0) {
@@ -474,6 +554,37 @@ public class RoleDescriptor implements ToXContentObject {
                 .build();
     }
 
+    private static ApplicationResourcePrivileges[] parseApplicationPrivileges(String roleName, XContentParser parser)
+            throws IOException {
+        if (parser.currentToken() != XContentParser.Token.START_ARRAY) {
+            throw new ElasticsearchParseException("failed to parse application privileges for role [{}]. expected field [{}] value " +
+                    "to be an array, but found [{}] instead", roleName, parser.currentName(), parser.currentToken());
+        }
+        List<ApplicationResourcePrivileges> privileges = new ArrayList<>();
+        while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+            privileges.add(parseApplicationPrivilege(roleName, parser));
+        }
+        return privileges.toArray(new ApplicationResourcePrivileges[privileges.size()]);
+    }
+
+    private static ApplicationResourcePrivileges parseApplicationPrivilege(String roleName, XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        if (token != XContentParser.Token.START_OBJECT) {
+            throw new ElasticsearchParseException("failed to parse application privileges for role [{}]. expected field [{}] value to " +
+                    "be an array of objects, but found an array element of type [{}]", roleName, parser.currentName(), token);
+        }
+        final ApplicationResourcePrivileges.Builder builder = ApplicationResourcePrivileges.PARSER.parse(parser, null);
+        if (builder.hasResources() == false) {
+            throw new ElasticsearchParseException("failed to parse application privileges for role [{}]. missing required [{}] field",
+                    roleName, Fields.RESOURCES.getPreferredName());
+        }
+        if (builder.hasPrivileges() == false) {
+            throw new ElasticsearchParseException("failed to parse application privileges for role [{}]. missing required [{}] field",
+                    roleName, Fields.PRIVILEGES.getPreferredName());
+        }
+        return builder.build();
+    }
+
     /**
      * A class representing permissions for a group of indices mapped to
      * privileges, field permissions, and a query.
@@ -695,14 +806,176 @@ public class RoleDescriptor implements ToXContentObject {
         }
     }
 
+    public static class ApplicationResourcePrivileges implements ToXContentObject, Streamable {
+
+        private static final ApplicationResourcePrivileges[] NONE = new ApplicationResourcePrivileges[0];
+        private static final ObjectParser<ApplicationResourcePrivileges.Builder, Void> PARSER = new ObjectParser<>("application",
+                ApplicationResourcePrivileges::builder);
+
+        static {
+            PARSER.declareString(Builder::application, Fields.APPLICATION);
+            PARSER.declareStringArray(Builder::privileges, Fields.PRIVILEGES);
+            PARSER.declareStringArray(Builder::resources, Fields.RESOURCES);
+        }
+
+        private String application;
+        private String[] privileges;
+        private String[] resources;
+
+        private ApplicationResourcePrivileges() {
+        }
+
+        public static Builder builder() {
+            return new Builder();
+        }
+
+        public String getApplication() {
+            return application;
+        }
+
+        public String[] getResources() {
+            return this.resources;
+        }
+
+        public String[] getPrivileges() {
+            return this.privileges;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder(getClass().getSimpleName())
+                    .append("[application=")
+                    .append(application)
+                    .append(", privileges=[")
+                    .append(Strings.arrayToCommaDelimitedString(privileges))
+                    .append("], resources=[")
+                    .append(Strings.arrayToCommaDelimitedString(resources))
+                    .append("]]");
+            return sb.toString();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || this.getClass() != o.getClass()) {
+                return false;
+            }
+
+            ApplicationResourcePrivileges that = (ApplicationResourcePrivileges) o;
+
+            return Objects.equals(this.application, that.application)
+                && Arrays.equals(this.resources, that.resources)
+                && Arrays.equals(this.privileges, that.privileges);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Arrays.hashCode(resources);
+            result = 31 * result + Arrays.hashCode(privileges);
+            return result;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(Fields.APPLICATION.getPreferredName(), application);
+            builder.array(Fields.PRIVILEGES.getPreferredName(), privileges);
+            builder.array(Fields.RESOURCES.getPreferredName(), resources);
+            return builder.endObject();
+        }
+
+        public static ApplicationResourcePrivileges createFrom(StreamInput in) throws IOException {
+            ApplicationResourcePrivileges ip = new ApplicationResourcePrivileges();
+            ip.readFrom(in);
+            return ip;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            this.application = in.readString();
+            this.privileges = in.readStringArray();
+            this.resources = in.readStringArray();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(application);
+            out.writeStringArray(privileges);
+            out.writeStringArray(resources);
+        }
+
+        public static void write(StreamOutput out, ApplicationResourcePrivileges privileges) throws IOException {
+            privileges.writeTo(out);
+        }
+
+        public static class Builder {
+
+            private ApplicationResourcePrivileges applicationPrivileges = new ApplicationResourcePrivileges();
+
+            private Builder() {
+            }
+
+            public Builder application(String appName) {
+                applicationPrivileges.application = appName;
+                return this;
+            }
+
+            public Builder resources(String... resources) {
+                applicationPrivileges.resources = resources;
+                return this;
+            }
+
+            public Builder resources(List<String> resources) {
+                return resources(resources.toArray(new String[resources.size()]));
+            }
+
+            public Builder privileges(String... privileges) {
+                applicationPrivileges.privileges = privileges;
+                return this;
+            }
+
+            public Builder privileges(Collection<String> privileges) {
+                return privileges(privileges.toArray(new String[privileges.size()]));
+            }
+
+            public boolean hasResources() {
+                return applicationPrivileges.resources != null;
+            }
+
+            public boolean hasPrivileges() {
+                return applicationPrivileges.privileges != null;
+            }
+
+            public ApplicationResourcePrivileges build() {
+                if (Strings.isNullOrEmpty(applicationPrivileges.application)) {
+                    throw new IllegalArgumentException("application privileges must have an application name");
+                }
+                if (applicationPrivileges.privileges == null || applicationPrivileges.privileges.length == 0) {
+                    throw new IllegalArgumentException("application privileges must define at least one privilege");
+                }
+                if (applicationPrivileges.resources == null || applicationPrivileges.resources.length == 0) {
+                    throw new IllegalArgumentException("application privileges must refer to at least one resource");
+                }
+                return applicationPrivileges;
+            }
+
+        }
+    }
+
     public interface Fields {
         ParseField CLUSTER = new ParseField("cluster");
+        ParseField GLOBAL = new ParseField("global");
         ParseField INDEX = new ParseField("index");
         ParseField INDICES = new ParseField("indices");
+        ParseField APPLICATIONS = new ParseField("applications");
         ParseField RUN_AS = new ParseField("run_as");
         ParseField NAMES = new ParseField("names");
+        ParseField RESOURCES = new ParseField("resources");
         ParseField QUERY = new ParseField("query");
         ParseField PRIVILEGES = new ParseField("privileges");
+        ParseField APPLICATION = new ParseField("application");
         ParseField FIELD_PERMISSIONS = new ParseField("field_security");
         ParseField FIELD_PERMISSIONS_2X = new ParseField("fields");
         ParseField GRANT_FIELDS = new ParseField("grant");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
new file mode 100644
index 00000000000..8f1e78a4663
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.support.Automatons;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+
+/**
+ * A permission that is based on privileges for application (non elasticsearch) capabilities
+ */
+public final class ApplicationPermission {
+
+    public static final ApplicationPermission NONE = new ApplicationPermission(Collections.emptyList());
+
+    private final Logger logger;
+    private final List<PermissionEntry> permissions;
+
+    /**
+     * @param privilegesAndResources A list of (privilege, resources). Each element in the {@link List} is a {@link Tuple} containing
+     *                               a single {@link ApplicationPrivilege} and the {@link Set} of resources to which that privilege is
+     *                               applied. The resources are treated as a wildcard {@link Automatons#pattern}.
+     */
+    ApplicationPermission(List<Tuple<ApplicationPrivilege, Set<String>>> privilegesAndResources) {
+        this.logger = Loggers.getLogger(getClass());
+        Map<ApplicationPrivilege, PermissionEntry> permissionsByPrivilege = new HashMap<>();
+        privilegesAndResources.forEach(tup -> permissionsByPrivilege.compute(tup.v1(), (k, existing) -> {
+            final Automaton patterns = Automatons.patterns(tup.v2());
+            if (existing == null) {
+                return new PermissionEntry(k, patterns);
+            } else {
+                return new PermissionEntry(k, Automatons.unionAndMinimize(Arrays.asList(existing.resources, patterns)));
+            }
+        }));
+        this.permissions = Collections.unmodifiableList(new ArrayList<>(permissionsByPrivilege.values()));
+    }
+
+    /**
+     * Determines whether this permission grants the specified privilege on the given resource.
+     * <p>
+     * An {@link ApplicationPermission} consists of a sequence of permission entries, where each entry contains a single
+     * {@link ApplicationPrivilege} and one or more resource patterns.
+     * </p>
+     * <p>
+     * This method returns {@code true} if, one or more of those entries meet the following criteria
+     * </p>
+     * <ul>
+     * <li>The entry's application, when interpreted as an {@link Automaton} {@link Automatons#pattern(String) pattern} matches the
+     * application given in the argument (interpreted as a raw string)
+     * </li>
+     * <li>The {@link ApplicationPrivilege#getAutomaton automaton that defines the entry's actions} entirely covers the
+     * automaton given in the argument (that is, the argument is a subset of the entry's automaton)
+     * </li>
+     * <li>The entry's resources, when interpreted as an {@link Automaton} {@link Automatons#patterns(String...)} set of patterns} entirely
+     * covers the resource given in the argument (also interpreted as an {@link Automaton} {@link Automatons#pattern(String) pattern}.
+     * </li>
+     * </ul>
+     */
+    public boolean grants(ApplicationPrivilege other, String resource) {
+        Automaton resourceAutomaton = Automatons.patterns(resource);
+        final boolean matched = permissions.stream().anyMatch(e -> e.grants(other, resourceAutomaton));
+        logger.trace("Permission [{}] {} grant [{} , {}]", this, matched ? "does" : "does not", other, resource);
+        return matched;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{privileges=" + permissions + "}";
+    }
+
+    private static class PermissionEntry {
+        private final ApplicationPrivilege privilege;
+        private final Predicate<String> application;
+        private final Automaton resources;
+
+        private PermissionEntry(ApplicationPrivilege privilege, Automaton resources) {
+            this.privilege = privilege;
+            this.application = Automatons.predicate(privilege.getApplication());
+            this.resources = resources;
+        }
+
+        private boolean grants(ApplicationPrivilege other, Automaton resource) {
+            return this.application.test(other.getApplication())
+                && Operations.isEmpty(privilege.getAutomaton()) == false
+                && Operations.subsetOf(other.getAutomaton(), privilege.getAutomaton())
+                && Operations.subsetOf(resource, this.resources);
+        }
+
+        @Override
+        public String toString() {
+            return privilege.toString();
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
index 7c990bd735a..370fd70b169 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
@@ -5,30 +5,97 @@
  */
 package org.elasticsearch.xpack.core.security.authz.permission;
 
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 
+import java.util.Collection;
+import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 /**
- * A permission that is based on privileges for cluster wide actions
+ * A permission that is based on privileges for cluster wide actions, with the optional ability to inspect the request object
  */
-public final class ClusterPermission {
-
-    public static final ClusterPermission NONE = new ClusterPermission(ClusterPrivilege.NONE);
-
+public abstract class ClusterPermission {
     private final ClusterPrivilege privilege;
-    private final Predicate<String> predicate;
 
     ClusterPermission(ClusterPrivilege privilege) {
         this.privilege = privilege;
-        this.predicate = privilege.predicate();
     }
 
     public ClusterPrivilege privilege() {
         return privilege;
     }
 
-    public boolean check(String action) {
-        return predicate.test(action);
+    public abstract boolean check(String action, TransportRequest request);
+
+    /**
+     * A permission that is based solely on cluster privileges and does not consider request state
+     */
+    public static class SimpleClusterPermission extends ClusterPermission {
+
+        public static final SimpleClusterPermission NONE = new SimpleClusterPermission(ClusterPrivilege.NONE);
+
+        private final Predicate<String> predicate;
+
+        SimpleClusterPermission(ClusterPrivilege privilege) {
+            super(privilege);
+            this.predicate = privilege.predicate();
+        }
+
+        @Override
+        public boolean check(String action, TransportRequest request) {
+            return predicate.test(action);
+        }
+    }
+
+    /**
+     * A permission that makes use of both cluster privileges and request inspection
+     */
+    public static class ConditionalClusterPermission extends ClusterPermission {
+        private final Predicate<String> actionPredicate;
+        private final Predicate<TransportRequest> requestPredicate;
+
+        public ConditionalClusterPermission(ConditionalClusterPrivilege conditionalPrivilege) {
+            this(conditionalPrivilege.getPrivilege(), conditionalPrivilege.getRequestPredicate());
+        }
+
+        public ConditionalClusterPermission(ClusterPrivilege privilege, Predicate<TransportRequest> requestPredicate) {
+            super(privilege);
+            this.actionPredicate = privilege.predicate();
+            this.requestPredicate = requestPredicate;
+        }
+
+        @Override
+        public boolean check(String action, TransportRequest request) {
+            return actionPredicate.test(action) && requestPredicate.test(request);
+        }
+    }
+
+    /**
+     * A permission that composes a number of other cluster permissions
+     */
+    public static class CompositeClusterPermission extends ClusterPermission {
+        private final Collection<ClusterPermission> children;
+
+        public CompositeClusterPermission(Collection<ClusterPermission> children) {
+            super(buildPrivilege(children));
+            this.children = children;
+        }
+
+        private static ClusterPrivilege buildPrivilege(Collection<ClusterPermission> children) {
+            final Set<String> names = children.stream()
+                .map(ClusterPermission::privilege)
+                .map(ClusterPrivilege::name)
+                .flatMap(Set::stream)
+                .collect(Collectors.toSet());
+            return ClusterPrivilege.get(names);
+        }
+
+        @Override
+        public boolean check(String action, TransportRequest request) {
+            return children.stream().anyMatch(p -> p.check(action, request));
+        }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
index a850a4a16f6..fd7eb486419 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
@@ -8,14 +8,18 @@ package org.elasticsearch.xpack.core.security.authz.permission;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -29,12 +33,14 @@ public final class Role {
     private final String[] names;
     private final ClusterPermission cluster;
     private final IndicesPermission indices;
+    private final ApplicationPermission application;
     private final RunAsPermission runAs;
 
-    Role(String[] names, ClusterPermission cluster, IndicesPermission indices, RunAsPermission runAs) {
+    Role(String[] names, ClusterPermission cluster, IndicesPermission indices, ApplicationPermission application, RunAsPermission runAs) {
         this.names = names;
         this.cluster = Objects.requireNonNull(cluster);
         this.indices = Objects.requireNonNull(indices);
+        this.application = Objects.requireNonNull(application);
         this.runAs = Objects.requireNonNull(runAs);
     }
 
@@ -50,6 +56,10 @@ public final class Role {
         return indices;
     }
 
+    public ApplicationPermission application() {
+        return application;
+    }
+
     public RunAsPermission runAs() {
         return runAs;
     }
@@ -70,7 +80,7 @@ public final class Role {
     public IndicesAccessControl authorize(String action, Set<String> requestedIndicesOrAliases, MetaData metaData,
                                           FieldPermissionsCache fieldPermissionsCache) {
         Map<String, IndicesAccessControl.IndexAccessControl> indexPermissions = indices.authorize(
-                action, requestedIndicesOrAliases, metaData, fieldPermissionsCache
+            action, requestedIndicesOrAliases, metaData, fieldPermissionsCache
         );
 
         // At least one role / indices permission set need to match with all the requested indices/aliases:
@@ -87,9 +97,10 @@ public final class Role {
     public static class Builder {
 
         private final String[] names;
-        private ClusterPermission cluster = ClusterPermission.NONE;
+        private ClusterPermission cluster = ClusterPermission.SimpleClusterPermission.NONE;
         private RunAsPermission runAs = RunAsPermission.NONE;
         private List<IndicesPermission.Group> groups = new ArrayList<>();
+        private List<Tuple<ApplicationPrivilege, Set<String>>> applicationPrivs = new ArrayList<>();
 
         private Builder(String[] names) {
             this.names = names;
@@ -97,20 +108,44 @@ public final class Role {
 
         private Builder(RoleDescriptor rd, @Nullable FieldPermissionsCache fieldPermissionsCache) {
             this.names = new String[] { rd.getName() };
-            if (rd.getClusterPrivileges().length == 0) {
-                cluster = ClusterPermission.NONE;
-            } else {
-                this.cluster(ClusterPrivilege.get(Sets.newHashSet(rd.getClusterPrivileges())));
-            }
+            cluster(Sets.newHashSet(rd.getClusterPrivileges()), Arrays.asList(rd.getConditionalClusterPrivileges()));
             groups.addAll(convertFromIndicesPrivileges(rd.getIndicesPrivileges(), fieldPermissionsCache));
+
+            final RoleDescriptor.ApplicationResourcePrivileges[] applicationPrivileges = rd.getApplicationPrivileges();
+            for (int i = 0; i < applicationPrivileges.length; i++) {
+                applicationPrivs.add(convertApplicationPrivilege(rd.getName(), i, applicationPrivileges[i]));
+            }
+
             String[] rdRunAs = rd.getRunAs();
             if (rdRunAs != null && rdRunAs.length > 0) {
                 this.runAs(new Privilege(Sets.newHashSet(rdRunAs), rdRunAs));
             }
         }
 
+        public Builder cluster(Set<String> privilegeNames, Iterable<ConditionalClusterPrivilege> conditionalClusterPrivileges) {
+            List<ClusterPermission> clusterPermissions = new ArrayList<>();
+            if (privilegeNames.isEmpty() == false) {
+                clusterPermissions.add(new ClusterPermission.SimpleClusterPermission(ClusterPrivilege.get(privilegeNames)));
+            }
+            for (ConditionalClusterPrivilege ccp : conditionalClusterPrivileges) {
+                clusterPermissions.add(new ClusterPermission.ConditionalClusterPermission(ccp));
+            }
+            if (clusterPermissions.isEmpty()) {
+                this.cluster = ClusterPermission.SimpleClusterPermission.NONE;
+            } else if (clusterPermissions.size() == 1) {
+                this.cluster = clusterPermissions.get(0);
+            } else {
+                this.cluster = new ClusterPermission.CompositeClusterPermission(clusterPermissions);
+            }
+            return this;
+        }
+
+        /**
+         * @deprecated Use {@link #cluster(Set, Iterable)}
+         */
+        @Deprecated
         public Builder cluster(ClusterPrivilege privilege) {
-            cluster = new ClusterPermission(privilege);
+            cluster = new ClusterPermission.SimpleClusterPermission(privilege);
             return this;
         }
 
@@ -129,10 +164,17 @@ public final class Role {
             return this;
         }
 
+        public Builder addApplicationPrivilege(ApplicationPrivilege privilege, Set<String> resources) {
+            applicationPrivs.add(new Tuple<>(privilege, resources));
+            return this;
+        }
+
         public Role build() {
             IndicesPermission indices = groups.isEmpty() ? IndicesPermission.NONE :
-                    new IndicesPermission(groups.toArray(new IndicesPermission.Group[groups.size()]));
-            return new Role(names, cluster, indices, runAs);
+                new IndicesPermission(groups.toArray(new IndicesPermission.Group[groups.size()]));
+            final ApplicationPermission applicationPermission
+                = applicationPrivs.isEmpty() ? ApplicationPermission.NONE : new ApplicationPermission(applicationPrivs);
+            return new Role(names, cluster, indices, applicationPermission, runAs);
         }
 
         static List<IndicesPermission.Group> convertFromIndicesPrivileges(RoleDescriptor.IndicesPrivileges[] indicesPrivileges,
@@ -144,16 +186,24 @@ public final class Role {
                     fieldPermissions = fieldPermissionsCache.getFieldPermissions(privilege.getGrantedFields(), privilege.getDeniedFields());
                 } else {
                     fieldPermissions = new FieldPermissions(
-                            new FieldPermissionsDefinition(privilege.getGrantedFields(), privilege.getDeniedFields()));
+                        new FieldPermissionsDefinition(privilege.getGrantedFields(), privilege.getDeniedFields()));
                 }
                 final Set<BytesReference> query = privilege.getQuery() == null ? null : Collections.singleton(privilege.getQuery());
                 list.add(new IndicesPermission.Group(IndexPrivilege.get(Sets.newHashSet(privilege.getPrivileges())),
-                        fieldPermissions,
-                        query,
-                        privilege.getIndices()));
+                    fieldPermissions,
+                    query,
+                    privilege.getIndices()));
 
             }
             return list;
         }
+
+        static Tuple<ApplicationPrivilege, Set<String>> convertApplicationPrivilege(String role, int index,
+                                                                                    RoleDescriptor.ApplicationResourcePrivileges arp) {
+            return new Tuple<>(new ApplicationPrivilege(arp.getApplication(),
+                "role." + role.replaceAll("[^a-zA-Z0-9]", "") + "." + index,
+                arp.getPrivileges()
+            ), Sets.newHashSet(arp.getResources()));
+        }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilege.java
new file mode 100644
index 00000000000..13db17a63bb
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilege.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import org.elasticsearch.common.Strings;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * An application privilege has an application name (e.g. {@code "my-app"}) that identifies an application (that exists
+ * outside of elasticsearch), a privilege name (e.g. {@code "admin}) that is meaningful to that application, and one or
+ * more "action patterns" (e.g {@code "admin/user/*", "admin/team/*"}).
+ * Action patterns must contain at least one special character from ({@code /}, {@code :}, {@code *}) to distinguish them
+ * from privilege names.
+ * The action patterns are entirely optional - many application will find that simple "privilege names" are sufficient, but
+ * they allow applications to define high level abstract privileges that map to multiple low level capabilities.
+ */
+public final class ApplicationPrivilege extends Privilege {
+
+    private static final Pattern VALID_APPLICATION_PREFIX = Pattern.compile("^[a-z][A-Za-z0-9]*$");
+    private static final Pattern WHITESPACE = Pattern.compile("[\\v\\h]");
+    private static final Pattern VALID_NAME = Pattern.compile("^[a-z][a-zA-Z0-9_.-]*$");
+
+    /**
+     * A name or action must be composed of printable, visible ASCII characters.
+     * That is: letters, numbers &amp; symbols, but no whitespace.
+     */
+    private static final Pattern VALID_NAME_OR_ACTION = Pattern.compile("^\\p{Graph}*$");
+
+    public static final Function<String, ApplicationPrivilege> NONE = app -> new ApplicationPrivilege(app, "none", new String[0]);
+
+    private final String application;
+    private final String[] patterns;
+
+    public ApplicationPrivilege(String application, String privilegeName, String... patterns) {
+        this(application, Collections.singleton(privilegeName), patterns);
+    }
+
+    public ApplicationPrivilege(String application, Set<String> name, String... patterns) {
+        super(name, patterns);
+        this.application = application;
+        this.patterns = patterns;
+    }
+
+    public String getApplication() {
+        return application;
+    }
+
+    // Package level for testing
+    String[] getPatterns() {
+        return patterns;
+    }
+
+    /**
+     * Validate that the provided application name is valid, and throws an exception otherwise
+     *
+     * @throws IllegalArgumentException if the name is not valid
+     */
+    public static void validateApplicationName(String application) {
+        validateApplicationName(application, false);
+    }
+
+    /**
+     * Validate that the provided name is a valid application, or a wildcard pattern for an application and throws an exception otherwise
+     *
+     * @throws IllegalArgumentException if the name is not valid
+     */
+    public static void validateApplicationNameOrWildcard(String application) {
+        validateApplicationName(application, true);
+    }
+
+    /**
+     * Validates that an application name matches the following rules:
+     * - consist of a "prefix", optionally followed by either "-" or "_" and a suffix
+     * - the prefix must begin with a lowercase ASCII letter
+     * - the prefix only contain ASCII letter or digits
+     * - the prefix must be at least 3 characters long
+     * - the suffix must only contain {@link Strings#validFileName valid filename} characters
+     * - no part of the name may contain whitespace
+     * If {@code allowWildcard} is true, then the names that end with a '*', and would match a valid
+     * application name are also accepted.
+     */
+    private static void validateApplicationName(String application, boolean allowWildcard) {
+        if (Strings.isEmpty(application)) {
+            throw new IllegalArgumentException("Application names cannot be blank");
+        }
+        final int asterisk = application.indexOf('*');
+        if (asterisk != -1) {
+            if (allowWildcard == false) {
+                throw new IllegalArgumentException("Application names may not contain '*' (found '" + application + "')");
+            }
+            if(application.equals("*")) {
+                // this is allowed and short-circuiting here makes the later validation simpler
+                return;
+            }
+            if (asterisk != application.length() - 1) {
+                throw new IllegalArgumentException("Application name patterns only support trailing wildcards (found '" + application
+                    + "')");
+            }
+        }
+        if (WHITESPACE.matcher(application).find()) {
+            throw new IllegalArgumentException("Application names may not contain whitespace (found '" + application + "')");
+        }
+
+        final String[] parts = application.split("[_-]", 2);
+        String prefix = parts[0];
+        if (prefix.endsWith("*")) {
+            prefix = prefix.substring(0, prefix.length() - 1);
+        }
+        if (VALID_APPLICATION_PREFIX.matcher(prefix).matches() == false) {
+            throw new IllegalArgumentException("An application name prefix must match the pattern " + VALID_APPLICATION_PREFIX.pattern()
+                + " (found '" + prefix + "')");
+        }
+        if (prefix.length() < 3 && asterisk == -1) {
+            throw new IllegalArgumentException("An application name prefix must be at least 3 characters long (found '" + prefix + "')");
+        }
+
+        if (parts.length > 1) {
+            final String suffix = parts[1];
+            if (Strings.validFileName(suffix) == false) {
+                throw new IllegalArgumentException("An application name suffix may not contain any of the characters '" +
+                    Strings.collectionToDelimitedString(Strings.INVALID_FILENAME_CHARS, "") + "' (found '" + suffix + "')");
+            }
+        }
+    }
+
+    /**
+     * Validate that the provided privilege name is valid, and throws an exception otherwise
+     *
+     * @throws IllegalArgumentException if the name is not valid
+     */
+    public static void validatePrivilegeName(String name) {
+        if (isValidPrivilegeName(name) == false) {
+            throw new IllegalArgumentException("Application privilege names must match the pattern " + VALID_NAME.pattern()
+                + " (found '" + name + "')");
+        }
+    }
+
+    private static boolean isValidPrivilegeName(String name) {
+        return VALID_NAME.matcher(name).matches();
+    }
+
+    /**
+     * Validate that the provided name is a valid privilege name or action name, and throws an exception otherwise
+     *
+     * @throws IllegalArgumentException if the name is not valid
+     */
+    public static void validatePrivilegeOrActionName(String name) {
+        if (VALID_NAME_OR_ACTION.matcher(name).matches() == false) {
+            throw new IllegalArgumentException("Application privilege names and actions must match the pattern "
+                + VALID_NAME_OR_ACTION.pattern() + " (found '" + name + "')");
+        }
+    }
+
+    /**
+     * Finds or creates an application privileges with the provided names.
+     * Each element in {@code name} may be the name of a stored privilege (to be resolved from {@code stored}, or a bespoke action pattern.
+     */
+    public static ApplicationPrivilege get(String application, Set<String> name, Collection<ApplicationPrivilegeDescriptor> stored) {
+        if (name.isEmpty()) {
+            return NONE.apply(application);
+        } else {
+            Map<String, ApplicationPrivilegeDescriptor> lookup = stored.stream()
+                .filter(apd -> apd.getApplication().equals(application))
+                .collect(Collectors.toMap(ApplicationPrivilegeDescriptor::getName, Function.identity()));
+            return resolve(application, name, lookup);
+        }
+    }
+
+    private static ApplicationPrivilege resolve(String application, Set<String> names, Map<String, ApplicationPrivilegeDescriptor> lookup) {
+        final int size = names.size();
+        if (size == 0) {
+            throw new IllegalArgumentException("empty set should not be used");
+        }
+
+        Set<String> actions = new HashSet<>();
+        Set<String> patterns = new HashSet<>();
+        for (String name : names) {
+            if (isValidPrivilegeName(name)) {
+                ApplicationPrivilegeDescriptor descriptor = lookup.get(name);
+                if (descriptor != null) {
+                    patterns.addAll(descriptor.getActions());
+                }
+            } else {
+                actions.add(name);
+            }
+        }
+
+        patterns.addAll(actions);
+        return new ApplicationPrivilege(application, names, patterns.toArray(new String[patterns.size()]));
+    }
+
+    @Override
+    public String toString() {
+        return application + ":" + super.toString() + "(" + Strings.arrayToCommaDelimitedString(patterns) + ")";
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + Objects.hashCode(application);
+        result = 31 * result + Arrays.hashCode(patterns);
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return super.equals(o)
+            && Objects.equals(this.application, ((ApplicationPrivilege) o).application)
+            && Arrays.equals(this.patterns, ((ApplicationPrivilege) o).patterns);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeDescriptor.java
new file mode 100644
index 00000000000..85d6aad3e35
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeDescriptor.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * An {@code ApplicationPrivilegeDescriptor} is a representation of a <em>stored</em> {@link ApplicationPrivilege}.
+ * A user (via a role) can be granted an application privilege by name (e.g. ("myapp", "read").
+ * In general, this privilege name will correspond to a pre-defined {@link ApplicationPrivilegeDescriptor}, which then
+ * is used to determine the set of actions granted by the privilege.
+ */
+public class ApplicationPrivilegeDescriptor implements ToXContentObject, Writeable {
+
+    public static final String DOC_TYPE_VALUE = "application-privilege";
+
+    private static final ObjectParser<Builder, Boolean> PARSER = new ObjectParser<>(DOC_TYPE_VALUE, Builder::new);
+
+    static {
+        PARSER.declareString(Builder::applicationName, Fields.APPLICATION);
+        PARSER.declareString(Builder::privilegeName, Fields.NAME);
+        PARSER.declareStringArray(Builder::actions, Fields.ACTIONS);
+        PARSER.declareObject(Builder::metadata, (parser, context) -> parser.map(), Fields.METADATA);
+        PARSER.declareField((parser, builder, allowType) -> builder.type(parser.text(), allowType), Fields.TYPE,
+            ObjectParser.ValueType.STRING);
+    }
+
+    private String application;
+    private String name;
+    private Set<String> actions;
+    private Map<String, Object> metadata;
+
+    public ApplicationPrivilegeDescriptor(String application, String name, Set<String> actions, Map<String, Object> metadata) {
+        this.application = Objects.requireNonNull(application);
+        this.name = Objects.requireNonNull(name);
+        this.actions = Collections.unmodifiableSet(actions);
+        this.metadata = Collections.unmodifiableMap(metadata);
+    }
+
+    public ApplicationPrivilegeDescriptor(StreamInput input) throws IOException {
+        this.application = input.readString();
+        this.name = input.readString();
+        this.actions = Collections.unmodifiableSet(input.readSet(StreamInput::readString));
+        this.metadata = Collections.unmodifiableMap(input.readMap());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(application);
+        out.writeString(name);
+        out.writeCollection(actions, StreamOutput::writeString);
+        out.writeMap(metadata);
+    }
+
+    public String getApplication() {
+        return application;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Set<String> getActions() {
+        return actions;
+    }
+
+    public Map<String, Object> getMetadata() {
+        return metadata;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return toXContent(builder, false);
+    }
+
+    public XContentBuilder toXContent(XContentBuilder builder, boolean includeTypeField) throws IOException {
+        builder.startObject()
+            .field(Fields.APPLICATION.getPreferredName(), application)
+            .field(Fields.NAME.getPreferredName(), name)
+            .field(Fields.ACTIONS.getPreferredName(), actions)
+            .field(Fields.METADATA.getPreferredName(), metadata);
+        if (includeTypeField) {
+            builder.field(Fields.TYPE.getPreferredName(), DOC_TYPE_VALUE);
+        }
+        return builder.endObject();
+    }
+
+    /**
+     * Construct a new {@link ApplicationPrivilegeDescriptor} from XContent.
+     *
+     * @param defaultApplication The application name to use if none is specified in the XContent body
+     * @param defaultName The privilege name to use if none is specified in the XContent body
+     * @param allowType If true, accept a "type" field (for which the value must match {@link #DOC_TYPE_VALUE});
+     */
+    public static ApplicationPrivilegeDescriptor parse(XContentParser parser, String defaultApplication, String defaultName,
+                                             boolean allowType) throws IOException {
+        final Builder builder = PARSER.parse(parser, allowType);
+        if (builder.applicationName == null) {
+            builder.applicationName(defaultApplication);
+        }
+        if (builder.privilegeName == null) {
+            builder.privilegeName(defaultName);
+        }
+        return builder.build();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        final ApplicationPrivilegeDescriptor that = (ApplicationPrivilegeDescriptor) o;
+        return Objects.equals(this.application, that.application) &&
+            Objects.equals(this.name, that.name) &&
+            Objects.equals(this.actions, that.actions) &&
+            Objects.equals(this.metadata, that.metadata);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(application, name, actions, metadata);
+    }
+
+    private static final class Builder {
+        private String applicationName;
+        private String privilegeName;
+        private Set<String> actions = Collections.emptySet();
+        private Map<String, Object> metadata = Collections.emptyMap();
+
+        private Builder applicationName(String applicationName) {
+            this.applicationName = applicationName;
+            return this;
+        }
+
+        private Builder privilegeName(String privilegeName) {
+            this.privilegeName = privilegeName;
+            return this;
+        }
+
+        private Builder actions(Collection<String> actions) {
+            this.actions = new HashSet<>(actions);
+            return this;
+        }
+
+        private Builder metadata(Map<String, Object> metadata) {
+            this.metadata = metadata;
+            return this;
+        }
+
+        private Builder type(String type, boolean allowed) {
+            if (allowed == false) {
+                throw new IllegalStateException("Field " + Fields.TYPE.getPreferredName() + " cannot be specified here");
+            }
+            if (ApplicationPrivilegeDescriptor.DOC_TYPE_VALUE.equals(type) == false) {
+                throw new IllegalStateException("XContent has wrong " + Fields.TYPE.getPreferredName() + " field " + type);
+            }
+            return this;
+        }
+
+        private ApplicationPrivilegeDescriptor build() {
+            return new ApplicationPrivilegeDescriptor(applicationName, privilegeName, actions, metadata);
+        }
+    }
+
+    public interface Fields {
+        ParseField APPLICATION = new ParseField("application");
+        ParseField NAME = new ParseField("name");
+        ParseField ACTIONS = new ParseField("actions");
+        ParseField METADATA = new ParseField("metadata");
+        ParseField TYPE = new ParseField("type");
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivilege.java
new file mode 100644
index 00000000000..dd89c2bda70
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivilege.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.transport.TransportRequest;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.function.Predicate;
+
+/**
+ * A ConditionalClusterPrivilege is a composition of a {@link ClusterPrivilege} (that determines which actions may be executed)
+ * with a {@link Predicate} for a {@link TransportRequest} (that determines which requests may be executed).
+ * The a given execution of an action is considered to be permitted if both the action and the request are permitted.
+ */
+public interface ConditionalClusterPrivilege extends NamedWriteable, ToXContentFragment {
+
+    /**
+     * The category under which this privilege should be rendered when output as XContent.
+     */
+    Category getCategory();
+
+    /**
+     * The action-level privilege that is required by this conditional privilege.
+     */
+    ClusterPrivilege getPrivilege();
+
+    /**
+     * The request-level privilege (as a {@link Predicate}) that is required by this conditional privilege.
+     */
+    Predicate<TransportRequest> getRequestPredicate();
+
+    /**
+     * A {@link ConditionalClusterPrivilege} should generate a fragment of {@code XContent}, which consists of
+     * a single field name, followed by its value (which may be an object, an array, or a simple value).
+     */
+    @Override
+    XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException;
+
+    /**
+     * Categories exist for to segment privileges for the purposes of rendering to XContent.
+     * {@link ConditionalClusterPrivileges#toXContent(XContentBuilder, Params, Collection)} builds one XContent
+     * object for a collection of {@link ConditionalClusterPrivilege} instances, with the top level fields built
+     * from the categories.
+     */
+    enum Category {
+        APPLICATION(new ParseField("application"));
+
+        public final ParseField field;
+
+        Category(ParseField field) {
+            this.field = field;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
new file mode 100644
index 00000000000..c068c77781b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParseException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.ApplicationPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege.Category;
+import org.elasticsearch.xpack.core.security.support.Automatons;
+import org.elasticsearch.xpack.core.security.xcontent.XContentUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Predicate;
+
+/**
+ * Static utility class for working with {@link ConditionalClusterPrivilege} instances
+ */
+public final class ConditionalClusterPrivileges {
+
+    public static final ConditionalClusterPrivilege[] EMPTY_ARRAY = new ConditionalClusterPrivilege[0];
+
+    private ConditionalClusterPrivileges() {
+    }
+
+    /**
+     * Utility method to read an array of {@link ConditionalClusterPrivilege} objects from a {@link StreamInput}
+     */
+    public static ConditionalClusterPrivilege[] readArray(StreamInput in) throws IOException {
+        return in.readArray(in1 ->
+            in1.readNamedWriteable(ConditionalClusterPrivilege.class), ConditionalClusterPrivilege[]::new);
+    }
+
+    /**
+     * Utility method to write an array of {@link ConditionalClusterPrivilege} objects to a {@link StreamOutput}
+     */
+    public static void writeArray(StreamOutput out, ConditionalClusterPrivilege[] privileges) throws IOException {
+        out.writeArray((out1, value) -> out1.writeNamedWriteable(value), privileges);
+    }
+
+    /**
+     * Writes a single object value to the {@code builder} that contains each of the provided privileges.
+     * The privileges are grouped according to their {@link ConditionalClusterPrivilege#getCategory() categories}
+     */
+    public static XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params,
+                                             Collection<ConditionalClusterPrivilege> privileges) throws IOException {
+        builder.startObject();
+        for (Category category : Category.values()) {
+            builder.startObject(category.field.getPreferredName());
+            for (ConditionalClusterPrivilege privilege : privileges) {
+                if (category == privilege.getCategory()) {
+                    privilege.toXContent(builder, params);
+                }
+            }
+            builder.endObject();
+        }
+        return builder.endObject();
+    }
+
+    /**
+     * Read a list of privileges from the parser. The parser should be positioned at the
+     * {@link XContentParser.Token#START_OBJECT} token for the privileges value
+     */
+    public static List<ConditionalClusterPrivilege> parse(XContentParser parser) throws IOException {
+        List<ConditionalClusterPrivilege> privileges = new ArrayList<>();
+
+        expectedToken(parser.currentToken(), parser, XContentParser.Token.START_OBJECT);
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            expectedToken(parser.currentToken(), parser, XContentParser.Token.FIELD_NAME);
+
+            expectFieldName(parser, Category.APPLICATION.field);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.START_OBJECT);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.FIELD_NAME);
+
+            expectFieldName(parser, ManageApplicationPrivileges.Fields.MANAGE);
+            privileges.add(ManageApplicationPrivileges.parse(parser));
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.END_OBJECT);
+        }
+
+        return privileges;
+    }
+
+    private static void expectedToken(XContentParser.Token read, XContentParser parser, XContentParser.Token expected) {
+        if (read != expected) {
+            throw new XContentParseException(parser.getTokenLocation(),
+                "failed to parse privilege. expected [" + expected + "] but found [" + read + "] instead");
+        }
+    }
+
+    private static void expectFieldName(XContentParser parser, ParseField... fields) throws IOException {
+        final String fieldName = parser.currentName();
+        if (Arrays.stream(fields).anyMatch(pf -> pf.match(fieldName, parser.getDeprecationHandler())) == false) {
+            throw new XContentParseException(parser.getTokenLocation(),
+                "failed to parse privilege. expected " + (fields.length == 1 ? "field name" : "one of") + " ["
+                    + Strings.arrayToCommaDelimitedString(fields) + "] but found [" + fieldName + "] instead");
+        }
+    }
+
+    /**
+     * The {@code ManageApplicationPrivileges} privilege is a {@link ConditionalClusterPrivilege} that grants the
+     * ability to execute actions related to the management of application privileges (Get, Put, Delete) for a subset
+     * of applications (identified by a wildcard-aware application-name).
+     */
+    public static class ManageApplicationPrivileges implements ConditionalClusterPrivilege {
+
+        private static final ClusterPrivilege PRIVILEGE = ClusterPrivilege.get(
+            Collections.singleton("cluster:admin/xpack/security/privilege/*")
+        );
+        public static final String WRITEABLE_NAME = "manage-application-privileges";
+
+        private final Set<String> applicationNames;
+        private final Predicate<String> applicationPredicate;
+        private final Predicate<TransportRequest> requestPredicate;
+
+        public ManageApplicationPrivileges(Set<String> applicationNames) {
+            this.applicationNames = Collections.unmodifiableSet(applicationNames);
+            this.applicationPredicate = Automatons.predicate(applicationNames);
+            this.requestPredicate = request -> {
+                if (request instanceof ApplicationPrivilegesRequest) {
+                    final ApplicationPrivilegesRequest privRequest = (ApplicationPrivilegesRequest) request;
+                    return privRequest.getApplicationNames().stream().allMatch(application -> applicationPredicate.test(application));
+                }
+                return false;
+            };
+        }
+
+        @Override
+        public Category getCategory() {
+            return Category.APPLICATION;
+        }
+
+        @Override
+        public ClusterPrivilege getPrivilege() {
+            return PRIVILEGE;
+        }
+
+        @Override
+        public Predicate<TransportRequest> getRequestPredicate() {
+            return this.requestPredicate;
+        }
+
+        public Collection<String> getApplicationNames() {
+            return Collections.unmodifiableCollection(this.applicationNames);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return WRITEABLE_NAME;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeCollection(this.applicationNames, StreamOutput::writeString);
+        }
+
+        public static ManageApplicationPrivileges createFrom(StreamInput in) throws IOException {
+            final Set<String> applications = in.readSet(StreamInput::readString);
+            return new ManageApplicationPrivileges(applications);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.field(Fields.MANAGE.getPreferredName(),
+                Collections.singletonMap(Fields.APPLICATIONS.getPreferredName(), applicationNames)
+            );
+        }
+
+        public static ManageApplicationPrivileges parse(XContentParser parser) throws IOException {
+            expectedToken(parser.currentToken(), parser, XContentParser.Token.FIELD_NAME);
+            expectFieldName(parser, Fields.MANAGE);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.START_OBJECT);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.FIELD_NAME);
+            expectFieldName(parser, Fields.APPLICATIONS);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.START_ARRAY);
+            final String[] applications = XContentUtils.readStringArray(parser, false);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.END_OBJECT);
+            return new ManageApplicationPrivileges(new LinkedHashSet<>(Arrays.asList(applications)));
+        }
+
+        @Override
+        public String toString() {
+            return "{" + getCategory() + ":" + Fields.MANAGE.getPreferredName() + ":" + Fields.APPLICATIONS.getPreferredName() + "="
+                + Strings.collectionToDelimitedString(applicationNames, ",") + "}";
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            final ManageApplicationPrivileges that = (ManageApplicationPrivileges) o;
+            return this.applicationNames.equals(that.applicationNames);
+        }
+
+        @Override
+        public int hashCode() {
+            return applicationNames.hashCode();
+        }
+
+        private interface Fields {
+            ParseField MANAGE = new ParseField("manage");
+            ParseField APPLICATIONS = new ParseField("applications");
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
index aeb448faa9c..0c593436365 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
@@ -9,6 +9,8 @@ import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges.ManageApplicationPrivileges;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 import org.elasticsearch.xpack.core.security.user.KibanaUser;
 import org.elasticsearch.xpack.core.security.user.UsernamesField;
@@ -27,8 +29,11 @@ public class ReservedRolesStore {
             new String[] { "all" },
             new RoleDescriptor.IndicesPrivileges[] {
                     RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").build()},
-            new String[] { "*" },
-            MetadataUtils.DEFAULT_RESERVED_METADATA);
+            new RoleDescriptor.ApplicationResourcePrivileges[] {
+                RoleDescriptor.ApplicationResourcePrivileges.builder().application("*").privileges("*").resources("*").build()
+            },
+            null, new String[] { "*" },
+            MetadataUtils.DEFAULT_RESERVED_METADATA, Collections.emptyMap());
     public static final Role SUPERUSER_ROLE = Role.builder(SUPERUSER_ROLE_DESCRIPTOR, null).build();
     private static final Map<String, RoleDescriptor> RESERVED_ROLES = initializeReservedRoles();
 
@@ -43,7 +48,11 @@ public class ReservedRolesStore {
                         MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put("kibana_user", new RoleDescriptor("kibana_user", null, new RoleDescriptor.IndicesPrivileges[] {
                         RoleDescriptor.IndicesPrivileges.builder().indices(".kibana*").privileges("manage", "read", "index", "delete")
-                                .build() }, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
+                            .build() }, new RoleDescriptor.ApplicationResourcePrivileges[] {
+                        RoleDescriptor.ApplicationResourcePrivileges.builder()
+                            .application("kibana-.kibana").resources("*").privileges("all").build() },
+                        null, null,
+                        MetadataUtils.DEFAULT_RESERVED_METADATA, null))
                 .put("monitoring_user", new RoleDescriptor("monitoring_user",
                         new String[] { "cluster:monitor/main" },
                         new RoleDescriptor.IndicesPrivileges[] {
@@ -70,13 +79,19 @@ public class ReservedRolesStore {
                         "kibana_dashboard_only_user",
                         null,
                         new RoleDescriptor.IndicesPrivileges[] {
-                                RoleDescriptor.IndicesPrivileges.builder()
-                                        .indices(".kibana*").privileges("read", "view_index_metadata").build()
+                            RoleDescriptor.IndicesPrivileges.builder()
+                                .indices(".kibana*").privileges("read", "view_index_metadata").build()
                         },
-                        null,
-                        MetadataUtils.DEFAULT_RESERVED_METADATA))
+                        new RoleDescriptor.ApplicationResourcePrivileges[] {
+                            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                            .application("kibana-.kibana").resources("*").privileges("read").build() },
+                        null, null,
+                        MetadataUtils.DEFAULT_RESERVED_METADATA,
+                        null))
                 .put(KibanaUser.ROLE_NAME, new RoleDescriptor(KibanaUser.ROLE_NAME,
-                        new String[] { "monitor", "manage_index_templates", MonitoringBulkAction.NAME, "manage_saml" },
+                        new String[] {
+                            "monitor", "manage_index_templates", MonitoringBulkAction.NAME, "manage_saml",
+                        },
                         new RoleDescriptor.IndicesPrivileges[] {
                                 RoleDescriptor.IndicesPrivileges.builder().indices(".kibana*", ".reporting-*").privileges("all").build(),
                                 RoleDescriptor.IndicesPrivileges.builder()
@@ -84,7 +99,9 @@ public class ReservedRolesStore {
                                 RoleDescriptor.IndicesPrivileges.builder()
                                         .indices(".management-beats").privileges("create_index", "read", "write").build()
                         },
-                        null, MetadataUtils.DEFAULT_RESERVED_METADATA))
+                        null,
+                        new ConditionalClusterPrivilege[] { new ManageApplicationPrivileges(Collections.singleton("kibana-*")) },
+                        null, MetadataUtils.DEFAULT_RESERVED_METADATA, null))
                 .put("logstash_system", new RoleDescriptor("logstash_system", new String[] { "monitor", MonitoringBulkAction.NAME},
                         null, null, MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put("beats_admin", new RoleDescriptor("beats_admin",
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
index 2668e62abbc..e1d3a2db8e9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
@@ -10,6 +10,12 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequestBuilder;
@@ -167,7 +173,9 @@ public class SecurityClient {
         client.execute(HasPrivilegesAction.INSTANCE, request, listener);
     }
 
-    /** User Management */
+    /**
+     * User Management
+     */
 
     public GetUsersRequestBuilder prepareGetUsers(String... usernames) {
         return new GetUsersRequestBuilder(client).usernames(usernames);
@@ -223,7 +231,9 @@ public class SecurityClient {
         client.execute(SetEnabledAction.INSTANCE, request, listener);
     }
 
-    /** Role Management */
+    /**
+     * Role Management
+     */
 
     public GetRolesRequestBuilder prepareGetRoles(String... names) {
         return new GetRolesRequestBuilder(client).names(names);
@@ -253,7 +263,9 @@ public class SecurityClient {
         client.execute(PutRoleAction.INSTANCE, request, listener);
     }
 
-    /** Role Mappings */
+    /**
+     * Role Mappings
+     */
 
     public GetRoleMappingsRequestBuilder prepareGetRoleMappings(String... names) {
         return new GetRoleMappingsRequestBuilder(client, GetRoleMappingsAction.INSTANCE)
@@ -275,6 +287,27 @@ public class SecurityClient {
                 .name(name);
     }
 
+    /* -- Application Privileges -- */
+    public GetPrivilegesRequestBuilder prepareGetPrivileges(String applicationName, String[] privileges) {
+        return new GetPrivilegesRequestBuilder(client, GetPrivilegesAction.INSTANCE).application(applicationName).privileges(privileges);
+    }
+
+    public PutPrivilegesRequestBuilder preparePutPrivilege(String applicationName, String privilegeName,
+                                                           BytesReference bytesReference, XContentType xContentType) throws IOException {
+        return new PutPrivilegesRequestBuilder(client, PutPrivilegesAction.INSTANCE)
+            .source(applicationName, privilegeName, bytesReference, xContentType);
+    }
+
+    public PutPrivilegesRequestBuilder preparePutPrivileges(BytesReference bytesReference, XContentType xContentType) throws IOException {
+        return new PutPrivilegesRequestBuilder(client, PutPrivilegesAction.INSTANCE).source(bytesReference, xContentType);
+    }
+
+    public DeletePrivilegesRequestBuilder prepareDeletePrivileges(String applicationName, String[] privileges) {
+        return new DeletePrivilegesRequestBuilder(client, DeletePrivilegesAction.INSTANCE)
+            .application(applicationName)
+            .privileges(privileges);
+    }
+
     public CreateTokenRequestBuilder prepareCreateToken() {
         return new CreateTokenRequestBuilder(client, CreateTokenAction.INSTANCE);
     }
@@ -298,7 +331,7 @@ public class SecurityClient {
         return builder;
     }
 
-    public void samlAuthenticate(SamlAuthenticateRequest request, ActionListener< SamlAuthenticateResponse> listener) {
+    public void samlAuthenticate(SamlAuthenticateRequest request, ActionListener<SamlAuthenticateResponse> listener) {
         client.execute(SamlAuthenticateAction.INSTANCE, request, listener);
     }
 
diff --git a/x-pack/plugin/core/src/main/resources/security-index-template.json b/x-pack/plugin/core/src/main/resources/security-index-template.json
index 778f44a93bf..dd17baf0474 100644
--- a/x-pack/plugin/core/src/main/resources/security-index-template.json
+++ b/x-pack/plugin/core/src/main/resources/security-index-template.json
@@ -91,6 +91,41 @@
             }
           }
         },
+        "applications": {
+          "type": "object",
+          "properties": {
+            "application": {
+              "type": "keyword"
+            },
+            "privileges": {
+              "type": "keyword"
+            },
+            "resources": {
+              "type": "keyword"
+            }
+          }
+        },
+        "application" : {
+          "type" : "keyword"
+        },
+        "global": {
+          "type": "object",
+          "properties": {
+            "application": {
+              "type": "object",
+              "properties": {
+                "manage": {
+                  "type": "object",
+                  "properties": {
+                    "applications": {
+                      "type": "keyword"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "name" : {
           "type" : "keyword"
         },
@@ -103,6 +138,9 @@
         "type" : {
           "type" : "keyword"
         },
+        "actions" : {
+          "type" : "keyword"
+        },
         "expiration_time" : {
           "type" : "date",
           "format" : "epoch_millis"
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java
index 9fd1d64323e..2a9041575df 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java
@@ -5,16 +5,16 @@
  */
 package org.elasticsearch.test;
 
+import org.hamcrest.CustomMatcher;
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
 import java.nio.file.Path;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
-import org.hamcrest.CustomMatcher;
-import org.hamcrest.Matcher;
-import org.hamcrest.Matchers;
-
 public class TestMatchers extends Matchers {
 
     public static Matcher<Path> pathExists(Path path, LinkOption... options) {
@@ -26,6 +26,19 @@ public class TestMatchers extends Matchers {
         };
     }
 
+    public static <T> Matcher<Predicate<T>> predicateMatches(T value) {
+        return new CustomMatcher<Predicate<T>>("Matches " + value) {
+            @Override
+            public boolean matches(Object item) {
+                if (Predicate.class.isInstance(item)) {
+                    return ((Predicate<T>) item).test(value);
+                } else {
+                    return false;
+                }
+            }
+        };
+    }
+
     public static Matcher<String> matchesPattern(String regex) {
         return matchesPattern(Pattern.compile(regex));
     }
@@ -34,16 +47,17 @@ public class TestMatchers extends Matchers {
         return predicate("Matches " + pattern.pattern(), String.class, pattern.asPredicate());
     }
 
-    private static <T> Matcher<T> predicate(String description, Class<T> type, Predicate<T> stringPredicate) {
+    private static <T> Matcher<T> predicate(String description, Class<T> type, Predicate<T> predicate) {
         return new CustomMatcher<T>(description) {
             @Override
             public boolean matches(Object item) {
                 if (type.isInstance(item)) {
-                    return stringPredicate.test(type.cast(item));
+                    return predicate.test(type.cast(item));
                 } else {
                     return false;
                 }
             }
         };
     }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestTests.java
new file mode 100644
index 00000000000..03232181f93
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestTests.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class DeletePrivilegesRequestTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final DeletePrivilegesRequest original = new DeletePrivilegesRequest(
+            randomAlphaOfLengthBetween(3, 8), generateRandomStringArray(5, randomIntBetween(3, 8), false, false));
+        original.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values()));
+
+        final BytesStreamOutput output = new BytesStreamOutput();
+        original.writeTo(output);
+        output.flush();
+        final DeletePrivilegesRequest copy = new DeletePrivilegesRequest();
+        copy.readFrom(output.bytes().streamInput());
+        assertThat(copy.application(), equalTo(original.application()));
+        assertThat(copy.privileges(), equalTo(original.privileges()));
+        assertThat(copy.getRefreshPolicy(), equalTo(original.getRefreshPolicy()));
+    }
+
+    public void testValidation() {
+        assertValidationFailure(new DeletePrivilegesRequest(null, null), "application name", "privileges");
+        assertValidationFailure(new DeletePrivilegesRequest("", null), "application name", "privileges");
+        assertValidationFailure(new DeletePrivilegesRequest(null, new String[0]), "application name", "privileges");
+        assertValidationFailure(new DeletePrivilegesRequest("", new String[0]), "application name", "privileges");
+        assertValidationFailure(new DeletePrivilegesRequest(null, new String[]{"all"}), "application name");
+        assertValidationFailure(new DeletePrivilegesRequest("", new String[]{"all"}), "application name");
+        assertValidationFailure(new DeletePrivilegesRequest("app", null), "privileges");
+        assertValidationFailure(new DeletePrivilegesRequest("app", new String[0]), "privileges");
+        assertValidationFailure(new DeletePrivilegesRequest("app", new String[]{""}), "privileges");
+
+        assertThat(new DeletePrivilegesRequest("app", new String[]{"all"}).validate(), nullValue());
+        assertThat(new DeletePrivilegesRequest("app", new String[]{"all", "some"}).validate(), nullValue());
+    }
+
+    private void assertValidationFailure(DeletePrivilegesRequest request, String... messages) {
+        final ActionRequestValidationException exception = request.validate();
+        assertThat(exception, notNullValue());
+        for (String message : messages) {
+            assertThat(exception.validationErrors(), Matchers.hasItem(containsString(message)));
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesResponseTests.java
new file mode 100644
index 00000000000..d490177c0ce
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesResponseTests.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DeletePrivilegesResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final DeletePrivilegesResponse original = new DeletePrivilegesResponse(
+            Arrays.asList(generateRandomStringArray(5, randomIntBetween(3, 8), false, true)));
+
+        final BytesStreamOutput output = new BytesStreamOutput();
+        original.writeTo(output);
+        output.flush();
+        final DeletePrivilegesResponse copy = new DeletePrivilegesResponse();
+        copy.readFrom(output.bytes().streamInput());
+        assertThat(copy.found(), equalTo(original.found()));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestTests.java
new file mode 100644
index 00000000000..4d67b82dfd4
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class GetPrivilegesRequestTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final GetPrivilegesRequest original = new GetPrivilegesRequest();
+        if (randomBoolean()) {
+            original.application(randomAlphaOfLengthBetween(3, 8));
+        }
+        original.privileges(generateRandomStringArray(3, 5, false, true));
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        final GetPrivilegesRequest copy = new GetPrivilegesRequest();
+        copy.readFrom(out.bytes().streamInput());
+
+        assertThat(original.application(), Matchers.equalTo(copy.application()));
+        assertThat(original.privileges(), Matchers.equalTo(copy.privileges()));
+    }
+
+    public void testValidation() {
+        assertThat(request(null).validate(), nullValue());
+        assertThat(request(null, "all").validate(), nullValue());
+        assertThat(request(null, "read", "write").validate(), nullValue());
+        assertThat(request("my_app").validate(), nullValue());
+        assertThat(request("my_app", "all").validate(), nullValue());
+        assertThat(request("my_app", "read", "write").validate(), nullValue());
+        final ActionRequestValidationException exception = request("my_app", ((String[]) null)).validate();
+        assertThat(exception, notNullValue());
+        assertThat(exception.validationErrors(), containsInAnyOrder("privileges cannot be null"));
+    }
+
+    private GetPrivilegesRequest request(String application, String... privileges) {
+        final GetPrivilegesRequest request = new GetPrivilegesRequest();
+        request.application(application);
+        request.privileges(privileges);
+        return request;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponseTests.java
new file mode 100644
index 00000000000..50c247967a7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponseTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Locale;
+
+public class GetPrivilegesResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        ApplicationPrivilegeDescriptor[] privileges = randomArray(6, ApplicationPrivilegeDescriptor[]::new, () ->
+            new ApplicationPrivilegeDescriptor(
+                randomAlphaOfLengthBetween(3, 8).toLowerCase(Locale.ROOT),
+                randomAlphaOfLengthBetween(3, 8).toLowerCase(Locale.ROOT),
+                Sets.newHashSet(randomArray(3, String[]::new, () -> randomAlphaOfLength(3).toLowerCase(Locale.ROOT) + "/*")),
+                Collections.emptyMap()
+            )
+        );
+        final GetPrivilegesResponse original = new GetPrivilegesResponse(privileges);
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        final GetPrivilegesResponse copy = new GetPrivilegesResponse();
+        copy.readFrom(out.bytes().streamInput());
+
+        assertThat(copy.privileges(), Matchers.equalTo(original.privileges()));
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestTests.java
new file mode 100644
index 00000000000..e258efd04c5
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestTests.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Locale;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class PutPrivilegesRequestTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final PutPrivilegesRequest original = request(randomArray(8, ApplicationPrivilegeDescriptor[]::new,
+            () -> new ApplicationPrivilegeDescriptor(
+                randomAlphaOfLengthBetween(3, 8).toLowerCase(Locale.ROOT),
+                randomAlphaOfLengthBetween(3, 8).toLowerCase(Locale.ROOT),
+                Sets.newHashSet(randomArray(3, String[]::new, () -> randomAlphaOfLength(3).toLowerCase(Locale.ROOT) + "/*")),
+                Collections.emptyMap()
+            )
+        ));
+        original.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values()));
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        final PutPrivilegesRequest copy = new PutPrivilegesRequest();
+        copy.readFrom(out.bytes().streamInput());
+
+        assertThat(original.getPrivileges(), Matchers.equalTo(copy.getPrivileges()));
+        assertThat(original.getRefreshPolicy(), Matchers.equalTo(copy.getRefreshPolicy()));
+    }
+
+    public void testValidation() {
+        // wildcard app name
+        final ApplicationPrivilegeDescriptor wildcardApp = descriptor("*", "all", "*");
+        assertValidationFailure(request(wildcardApp), "Application names may not contain");
+
+        // invalid priv names
+        final ApplicationPrivilegeDescriptor spaceName = descriptor("app", "r e a d", "read/*");
+        final ApplicationPrivilegeDescriptor numericName = descriptor("app", "7346", "read/*");
+        assertValidationFailure(request(spaceName), "Application privilege names must match");
+        assertValidationFailure(request(numericName), "Application privilege names must match");
+
+        // no actions
+        final ApplicationPrivilegeDescriptor nothing = descriptor("*", "nothing");
+        assertValidationFailure(request(nothing), "Application privileges must have at least one action");
+
+        // reserved metadata
+        final ApplicationPrivilegeDescriptor reservedMetadata = new ApplicationPrivilegeDescriptor("app", "all",
+            Collections.emptySet(), Collections.singletonMap("_notAllowed", true)
+        );
+        assertValidationFailure(request(reservedMetadata), "metadata keys may not start");
+
+        ApplicationPrivilegeDescriptor badAction = descriptor("app", "foo", randomFrom("data.read", "data_read", "data+read", "read"));
+        assertValidationFailure(request(badAction), "must contain one of");
+
+        // mixed
+        assertValidationFailure(request(wildcardApp, numericName, reservedMetadata, badAction),
+            "Application names may not contain", "Application privilege names must match", "metadata keys may not start",
+            "must contain one of");
+    }
+
+    private ApplicationPrivilegeDescriptor descriptor(String application, String name, String... actions) {
+        return new ApplicationPrivilegeDescriptor(application, name, Sets.newHashSet(actions), Collections.emptyMap());
+    }
+
+    private void assertValidationFailure(PutPrivilegesRequest request, String... messages) {
+        final ActionRequestValidationException exception = request.validate();
+        assertThat(exception, notNullValue());
+        for (String message : messages) {
+            assertThat(exception.validationErrors(), hasItem(containsString(message)));
+        }
+    }
+
+    private PutPrivilegesRequest request(ApplicationPrivilegeDescriptor... privileges) {
+        final PutPrivilegesRequest original = new PutPrivilegesRequest();
+
+        original.setPrivileges(Arrays.asList(privileges));
+        return original;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesResponseTests.java
new file mode 100644
index 00000000000..431d7f326ee
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesResponseTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class PutPrivilegesResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final int applicationCount = randomInt(3);
+        final Map<String, List<String>> map = new HashMap<>(applicationCount);
+        for (int i = 0; i < applicationCount; i++) {
+            map.put(randomAlphaOfLengthBetween(3, 8),
+                Arrays.asList(generateRandomStringArray(5, 6, false, true))
+            );
+        }
+        final PutPrivilegesResponse original = new PutPrivilegesResponse(map);
+
+        final BytesStreamOutput output = new BytesStreamOutput();
+        original.writeTo(output);
+        output.flush();
+        final PutPrivilegesResponse copy = new PutPrivilegesResponse();
+        copy.readFrom(output.bytes().streamInput());
+        assertThat(copy.created(), equalTo(original.created()));
+        assertThat(Strings.toString(copy), equalTo(Strings.toString(original)));
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java
new file mode 100644
index 00000000000..ae458cbb2f5
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.role;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class PutRoleRequestTests extends ESTestCase {
+
+    public void testValidationOfApplicationPrivileges() {
+        assertSuccessfulValidation(buildRequestWithApplicationPrivilege("app", new String[]{"read"}, new String[]{"*"}));
+        assertSuccessfulValidation(buildRequestWithApplicationPrivilege("app", new String[]{"action:login"}, new String[]{"/"}));
+        assertSuccessfulValidation(buildRequestWithApplicationPrivilege("*", new String[]{"data/read:user"}, new String[]{"user/123"}));
+
+        // Fail
+        assertValidationError("privilege names and actions must match the pattern",
+            buildRequestWithApplicationPrivilege("app", new String[]{"in valid"}, new String[]{"*"}));
+        assertValidationError("An application name prefix must match the pattern",
+            buildRequestWithApplicationPrivilege("000", new String[]{"all"}, new String[]{"*"}));
+        assertValidationError("An application name prefix must match the pattern",
+            buildRequestWithApplicationPrivilege("%*", new String[]{"all"}, new String[]{"*"}));
+    }
+
+    public void testSerialization() throws IOException {
+        final PutRoleRequest original = buildRandomRequest();
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        final PutRoleRequest copy = new PutRoleRequest();
+        final NamedWriteableRegistry registry = new NamedWriteableRegistry(new XPackClientPlugin(Settings.EMPTY).getNamedWriteables());
+        StreamInput in = new NamedWriteableAwareStreamInput(ByteBufferStreamInput.wrap(BytesReference.toBytes(out.bytes())), registry);
+        copy.readFrom(in);
+
+        assertThat(copy.roleDescriptor(), equalTo(original.roleDescriptor()));
+    }
+
+    public void testSerializationV63AndBefore() throws IOException {
+        final PutRoleRequest original = buildRandomRequest();
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_6_0, Version.V_6_3_2);
+        out.setVersion(version);
+        original.writeTo(out);
+
+        final PutRoleRequest copy = new PutRoleRequest();
+        final StreamInput in = out.bytes().streamInput();
+        in.setVersion(version);
+        copy.readFrom(in);
+
+        assertThat(copy.name(), equalTo(original.name()));
+        assertThat(copy.cluster(), equalTo(original.cluster()));
+        assertThat(copy.indices(), equalTo(original.indices()));
+        assertThat(copy.runAs(), equalTo(original.runAs()));
+        assertThat(copy.metadata(), equalTo(original.metadata()));
+        assertThat(copy.getRefreshPolicy(), equalTo(original.getRefreshPolicy()));
+
+        assertThat(copy.applicationPrivileges(), iterableWithSize(0));
+        assertThat(copy.conditionalClusterPrivileges(), arrayWithSize(0));
+    }
+
+    private void assertSuccessfulValidation(PutRoleRequest request) {
+        final ActionRequestValidationException exception = request.validate();
+        assertThat(exception, nullValue());
+    }
+
+    private void assertValidationError(String message, PutRoleRequest request) {
+        final ActionRequestValidationException exception = request.validate();
+        assertThat(exception, notNullValue());
+        assertThat(exception.validationErrors(), hasItem(containsString(message)));
+    }
+
+    private PutRoleRequest buildRequestWithApplicationPrivilege(String appName, String[] privileges, String[] resources) {
+        final PutRoleRequest request = new PutRoleRequest();
+        request.name("test");
+        final ApplicationResourcePrivileges privilege = ApplicationResourcePrivileges.builder()
+            .application(appName)
+            .privileges(privileges)
+            .resources(resources)
+            .build();
+        request.addApplicationPrivileges(new ApplicationResourcePrivileges[]{privilege});
+        return request;
+    }
+
+    private PutRoleRequest buildRandomRequest() {
+
+        final PutRoleRequest request = new PutRoleRequest();
+        request.name(randomAlphaOfLengthBetween(4, 9));
+
+        request.cluster(randomSubsetOf(Arrays.asList("monitor", "manage", "all", "manage_security", "manage_ml", "monitor_watcher"))
+            .toArray(Strings.EMPTY_ARRAY));
+
+        for (int i = randomIntBetween(0, 4); i > 0; i--) {
+            request.addIndex(
+                generateRandomStringArray(randomIntBetween(1, 3), randomIntBetween(3, 8), false, false),
+                randomSubsetOf(randomIntBetween(1, 2), "read", "write", "index", "all").toArray(Strings.EMPTY_ARRAY),
+                generateRandomStringArray(randomIntBetween(1, 3), randomIntBetween(3, 8), true),
+                generateRandomStringArray(randomIntBetween(1, 3), randomIntBetween(3, 8), true),
+                null
+            );
+        }
+
+        final Supplier<String> stringWithInitialLowercase = ()
+            -> randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(3, 12);
+        final ApplicationResourcePrivileges[] applicationPrivileges = new ApplicationResourcePrivileges[randomIntBetween(0, 5)];
+        for (int i = 0; i < applicationPrivileges.length; i++) {
+            applicationPrivileges[i] = ApplicationResourcePrivileges.builder()
+                .application(stringWithInitialLowercase.get())
+                .privileges(randomArray(1, 3, String[]::new, stringWithInitialLowercase))
+                .resources(generateRandomStringArray(5, randomIntBetween(3, 8), false, false))
+                .build();
+        }
+        request.addApplicationPrivileges(applicationPrivileges);
+
+        if (randomBoolean()) {
+            final String[] appNames = randomArray(1, 4, String[]::new, stringWithInitialLowercase);
+            request.conditionalCluster(new ConditionalClusterPrivileges.ManageApplicationPrivileges(Sets.newHashSet(appNames)));
+        }
+
+        request.runAs(generateRandomStringArray(4, 3, false, true));
+
+        final Map<String, Object> metadata = new HashMap<>();
+        for (String key : generateRandomStringArray(3, 5, false, true)) {
+            metadata.put(key, randomFrom(Boolean.TRUE, Boolean.FALSE, 1, 2, randomAlphaOfLengthBetween(2, 9)));
+        }
+        request.metadata(metadata);
+
+        request.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values()));
+        return request;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestTests.java
new file mode 100644
index 00000000000..f458311e685
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestTests.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.user;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class HasPrivilegesRequestTests extends ESTestCase {
+
+    public void testSerializationV7() throws IOException {
+        final HasPrivilegesRequest original = randomRequest();
+        final HasPrivilegesRequest copy = serializeAndDeserialize(original, Version.V_7_0_0_alpha1);
+
+        assertThat(copy.username(), equalTo(original.username()));
+        assertThat(copy.clusterPrivileges(), equalTo(original.clusterPrivileges()));
+        assertThat(copy.indexPrivileges(), equalTo(original.indexPrivileges()));
+        assertThat(copy.applicationPrivileges(), equalTo(original.applicationPrivileges()));
+    }
+
+    public void testSerializationV63() throws IOException {
+        final HasPrivilegesRequest original = randomRequest();
+        final HasPrivilegesRequest copy = serializeAndDeserialize(original, Version.V_6_3_0);
+
+        assertThat(copy.username(), equalTo(original.username()));
+        assertThat(copy.clusterPrivileges(), equalTo(original.clusterPrivileges()));
+        assertThat(copy.indexPrivileges(), equalTo(original.indexPrivileges()));
+        assertThat(copy.applicationPrivileges(), nullValue());
+    }
+
+    public void testValidateNullPrivileges() {
+        final HasPrivilegesRequest request = new HasPrivilegesRequest();
+        final ActionRequestValidationException exception = request.validate();
+        assertThat(exception, notNullValue());
+        assertThat(exception.validationErrors(), hasItem("clusterPrivileges must not be null"));
+        assertThat(exception.validationErrors(), hasItem("indexPrivileges must not be null"));
+        assertThat(exception.validationErrors(), hasItem("applicationPrivileges must not be null"));
+    }
+
+    public void testValidateEmptyPrivileges() {
+        final HasPrivilegesRequest request = new HasPrivilegesRequest();
+        request.clusterPrivileges(new String[0]);
+        request.indexPrivileges(new IndicesPrivileges[0]);
+        request.applicationPrivileges(new ApplicationResourcePrivileges[0]);
+        final ActionRequestValidationException exception = request.validate();
+        assertThat(exception, notNullValue());
+        assertThat(exception.validationErrors(), hasItem("must specify at least one privilege"));
+    }
+
+    public void testValidateNoWildcardApplicationPrivileges() {
+        final HasPrivilegesRequest request = new HasPrivilegesRequest();
+        request.clusterPrivileges(new String[0]);
+        request.indexPrivileges(new IndicesPrivileges[0]);
+        request.applicationPrivileges(new ApplicationResourcePrivileges[] {
+            ApplicationResourcePrivileges.builder().privileges("read").application("*").resources("item/1").build()
+        });
+        final ActionRequestValidationException exception = request.validate();
+        assertThat(exception, notNullValue());
+        assertThat(exception.validationErrors(), hasItem("Application names may not contain '*' (found '*')"));
+    }
+
+    private HasPrivilegesRequest serializeAndDeserialize(HasPrivilegesRequest original, Version version) throws IOException {
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.setVersion(version);
+        original.writeTo(out);
+
+        final HasPrivilegesRequest copy = new HasPrivilegesRequest();
+        final StreamInput in = out.bytes().streamInput();
+        in.setVersion(version);
+        copy.readFrom(in);
+        assertThat(in.read(), equalTo(-1));
+        return copy;
+    }
+
+    private HasPrivilegesRequest randomRequest() {
+        final HasPrivilegesRequest request = new HasPrivilegesRequest();
+        request.username(randomAlphaOfLength(8));
+
+        final List<String> clusterPrivileges = randomSubsetOf(Arrays.asList(ClusterPrivilege.MONITOR, ClusterPrivilege.MANAGE,
+            ClusterPrivilege.MANAGE_ML, ClusterPrivilege.MANAGE_SECURITY, ClusterPrivilege.MANAGE_PIPELINE, ClusterPrivilege.ALL))
+            .stream().flatMap(p -> p.name().stream()).collect(Collectors.toList());
+        request.clusterPrivileges(clusterPrivileges.toArray(Strings.EMPTY_ARRAY));
+
+        IndicesPrivileges[] indicesPrivileges = new IndicesPrivileges[randomInt(5)];
+        for (int i = 0; i < indicesPrivileges.length; i++) {
+            indicesPrivileges[i] = IndicesPrivileges.builder()
+                .privileges(randomFrom("read", "write", "create", "delete", "all"))
+                .indices(randomAlphaOfLengthBetween(2, 8) + (randomBoolean() ? "*" : ""))
+                .build();
+        }
+        request.indexPrivileges(indicesPrivileges);
+
+        final ApplicationResourcePrivileges[] appPrivileges = new ApplicationResourcePrivileges[randomInt(5)];
+        for (int i = 0; i < appPrivileges.length; i++) {
+            appPrivileges[i] = ApplicationResourcePrivileges.builder()
+                .application(randomAlphaOfLengthBetween(3, 8))
+                .resources(randomAlphaOfLengthBetween(5, 7) + (randomBoolean() ? "*" : ""))
+                .privileges(generateRandomStringArray(6, 7, false, false))
+                .build();
+        }
+        request.applicationPrivileges(appPrivileges);
+        return request;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermissionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermissionTests.java
new file mode 100644
index 00000000000..47a189b41f1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermissionTests.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ApplicationPermissionTests extends ESTestCase {
+
+    private List<ApplicationPrivilegeDescriptor> store = new ArrayList<>();
+
+    private ApplicationPrivilege app1All = storePrivilege("app1", "all", "*");
+    private ApplicationPrivilege app1Empty = storePrivilege("app1", "empty");
+    private ApplicationPrivilege app1Read = storePrivilege("app1", "read", "read/*");
+    private ApplicationPrivilege app1Write = storePrivilege("app1", "write", "write/*");
+    private ApplicationPrivilege app1Delete = storePrivilege("app1", "delete", "write/delete");
+    private ApplicationPrivilege app1Create = storePrivilege("app1", "create", "write/create");
+    private ApplicationPrivilege app2Read = storePrivilege("app2", "read", "read/*");
+
+    private ApplicationPrivilege storePrivilege(String app, String name, String... patterns) {
+        store.add(new ApplicationPrivilegeDescriptor(app, name, Sets.newHashSet(patterns), Collections.emptyMap()));
+        return new ApplicationPrivilege(app, name, patterns);
+    }
+
+    public void testCheckSimplePermission() {
+        final ApplicationPermission hasPermission = buildPermission(app1Write, "*");
+        assertThat(hasPermission.grants(app1Write, "*"), equalTo(true));
+        assertThat(hasPermission.grants(app1Write, "foo"), equalTo(true));
+        assertThat(hasPermission.grants(app1Delete, "*"), equalTo(true));
+        assertThat(hasPermission.grants(app1Create, "foo"), equalTo(true));
+
+        assertThat(hasPermission.grants(app1Read, "*"), equalTo(false));
+        assertThat(hasPermission.grants(app1Read, "foo"), equalTo(false));
+        assertThat(hasPermission.grants(app1All, "*"), equalTo(false));
+        assertThat(hasPermission.grants(app1All, "foo"), equalTo(false));
+    }
+
+    public void testNonePermission() {
+        final ApplicationPermission hasPermission = buildPermission(ApplicationPrivilege.NONE.apply("app1"), "*");
+        for (ApplicationPrivilege privilege : Arrays.asList(app1All, app1Empty, app1Create, app1Delete, app1Read, app1Write, app2Read)) {
+            assertThat("Privilege " + privilege + " on *", hasPermission.grants(privilege, "*"), equalTo(false));
+            final String resource = randomAlphaOfLengthBetween(1, 6);
+            assertThat("Privilege " + privilege + " on " + resource, hasPermission.grants(privilege, resource), equalTo(false));
+        }
+    }
+
+    public void testResourceMatching() {
+        final ApplicationPermission hasPermission = buildPermission(app1All, "dashboard/*", "audit/*", "user/12345");
+
+        assertThat(hasPermission.grants(app1Write, "*"), equalTo(false));
+        assertThat(hasPermission.grants(app1Write, "dashboard"), equalTo(false));
+        assertThat(hasPermission.grants(app1Write, "dashboard/999"), equalTo(true));
+
+        assertThat(hasPermission.grants(app1Create, "audit/2018-02-21"), equalTo(true));
+        assertThat(hasPermission.grants(app1Create, "report/2018-02-21"), equalTo(false));
+
+        assertThat(hasPermission.grants(app1Read, "user/12345"), equalTo(true));
+        assertThat(hasPermission.grants(app1Read, "user/67890"), equalTo(false));
+
+        assertThat(hasPermission.grants(app1All, "dashboard/999"), equalTo(true));
+        assertThat(hasPermission.grants(app1All, "audit/2018-02-21"), equalTo(true));
+        assertThat(hasPermission.grants(app1All, "user/12345"), equalTo(true));
+    }
+
+    public void testActionMatching() {
+        final ApplicationPermission hasPermission = buildPermission(app1Write, "allow/*");
+
+        final ApplicationPrivilege update = actionPrivilege("app1", "write/update");
+        assertThat(hasPermission.grants(update, "allow/1"), equalTo(true));
+        assertThat(hasPermission.grants(update, "deny/1"), equalTo(false));
+
+        final ApplicationPrivilege updateCreate = actionPrivilege("app1", "write/update", "write/create");
+        assertThat(hasPermission.grants(updateCreate, "allow/1"), equalTo(true));
+        assertThat(hasPermission.grants(updateCreate, "deny/1"), equalTo(false));
+
+        final ApplicationPrivilege manage = actionPrivilege("app1", "admin/manage");
+        assertThat(hasPermission.grants(manage, "allow/1"), equalTo(false));
+        assertThat(hasPermission.grants(manage, "deny/1"), equalTo(false));
+    }
+
+    public void testDoesNotMatchAcrossApplications() {
+        assertThat(buildPermission(app1Read, "*").grants(app1Read, "123"), equalTo(true));
+        assertThat(buildPermission(app1All, "*").grants(app1Read, "123"), equalTo(true));
+
+        assertThat(buildPermission(app1Read, "*").grants(app2Read, "123"), equalTo(false));
+        assertThat(buildPermission(app1All, "*").grants(app2Read, "123"), equalTo(false));
+    }
+
+    public void testMergedPermissionChecking() {
+        final ApplicationPrivilege app1ReadWrite = ApplicationPrivilege.get("app1", Sets.union(app1Read.name(), app1Write.name()), store);
+        final ApplicationPermission hasPermission = buildPermission(app1ReadWrite, "allow/*");
+
+        assertThat(hasPermission.grants(app1Read, "allow/1"), equalTo(true));
+        assertThat(hasPermission.grants(app1Write, "allow/1"), equalTo(true));
+
+        assertThat(hasPermission.grants(app1Read, "deny/1"), equalTo(false));
+        assertThat(hasPermission.grants(app1Write, "deny/1"), equalTo(false));
+
+        assertThat(hasPermission.grants(app1All, "allow/1"), equalTo(false));
+        assertThat(hasPermission.grants(app2Read, "allow/1"), equalTo(false));
+    }
+
+    private ApplicationPrivilege actionPrivilege(String appName, String... actions) {
+        return ApplicationPrivilege.get(appName, Sets.newHashSet(actions), Collections.emptyList());
+    }
+
+    private ApplicationPermission buildPermission(ApplicationPrivilege privilege, String... resources) {
+        return new ApplicationPermission(singletonList(new Tuple<>(privilege, Sets.newHashSet(resources))));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeDescriptorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeDescriptorTests.java
new file mode 100644
index 00000000000..9db998bd2d1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeDescriptorTests.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.hamcrest.Matchers;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.iterableWithSize;
+
+public class ApplicationPrivilegeDescriptorTests extends ESTestCase {
+
+    public void testEqualsAndHashCode() {
+        final ApplicationPrivilegeDescriptor privilege = randomPrivilege();
+        final EqualsHashCodeTestUtils.MutateFunction<ApplicationPrivilegeDescriptor> mutate = randomFrom(
+            orig -> new ApplicationPrivilegeDescriptor(
+                "x" + orig.getApplication(), orig.getName(), orig.getActions(), orig.getMetadata()),
+            orig -> new ApplicationPrivilegeDescriptor(
+                orig.getApplication(), "x" + orig.getName(), orig.getActions(), orig.getMetadata()),
+            orig -> new ApplicationPrivilegeDescriptor(
+                orig.getApplication(), orig.getName(), Collections.singleton("*"), orig.getMetadata()),
+            orig -> new ApplicationPrivilegeDescriptor(
+                orig.getApplication(), orig.getName(), orig.getActions(), Collections.singletonMap("mutate", -1L))
+        );
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(privilege,
+            original -> new ApplicationPrivilegeDescriptor(
+                original.getApplication(), original.getName(), original.getActions(), original.getMetadata()),
+            mutate
+        );
+    }
+
+    public void testSerialization() throws IOException {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            final ApplicationPrivilegeDescriptor original = randomPrivilege();
+            original.writeTo(out);
+            final ApplicationPrivilegeDescriptor clone = new ApplicationPrivilegeDescriptor(out.bytes().streamInput());
+            assertThat(clone, Matchers.equalTo(original));
+            assertThat(original, Matchers.equalTo(clone));
+        }
+    }
+
+    public void testXContentGenerationAndParsing() throws IOException {
+        final boolean includeTypeField = randomBoolean();
+
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            final XContentBuilder builder = new XContentBuilder(xContent, out);
+
+            final ApplicationPrivilegeDescriptor original = randomPrivilege();
+            if (includeTypeField) {
+                original.toXContent(builder, true);
+            } else if (randomBoolean()) {
+                original.toXContent(builder, false);
+            } else {
+                original.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            }
+            builder.flush();
+
+            final byte[] bytes = out.toByteArray();
+            try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, bytes)) {
+                final ApplicationPrivilegeDescriptor clone = ApplicationPrivilegeDescriptor.parse(parser,
+                    randomBoolean() ? randomAlphaOfLength(3) : null,
+                    randomBoolean() ? randomAlphaOfLength(3) : null,
+                    includeTypeField);
+                assertThat(clone, Matchers.equalTo(original));
+                assertThat(original, Matchers.equalTo(clone));
+            }
+        }
+    }
+
+    public void testParseXContentWithDefaultNames() throws IOException {
+        final String json = "{ \"actions\": [ \"data:read\" ], \"metadata\" : { \"num\": 1, \"bool\":false } }";
+        final XContent xContent = XContentType.JSON.xContent();
+        try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, json)) {
+            final ApplicationPrivilegeDescriptor privilege = ApplicationPrivilegeDescriptor.parse(parser, "my_app", "read", false);
+            assertThat(privilege.getApplication(), equalTo("my_app"));
+            assertThat(privilege.getName(), equalTo("read"));
+            assertThat(privilege.getActions(), contains("data:read"));
+            assertThat(privilege.getMetadata().entrySet(), iterableWithSize(2));
+            assertThat(privilege.getMetadata().get("num"), equalTo(1));
+            assertThat(privilege.getMetadata().get("bool"), equalTo(false));
+        }
+    }
+
+    public void testParseXContentWithoutUsingDefaultNames() throws IOException {
+        final String json = "{" +
+            "  \"application\": \"your_app\"," +
+            "  \"name\": \"write\"," +
+            "  \"actions\": [ \"data:write\" ]" +
+            "}";
+        final XContent xContent = XContentType.JSON.xContent();
+        try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, json)) {
+            final ApplicationPrivilegeDescriptor privilege = ApplicationPrivilegeDescriptor.parse(parser, "my_app", "read", false);
+            assertThat(privilege.getApplication(), equalTo("your_app"));
+            assertThat(privilege.getName(), equalTo("write"));
+            assertThat(privilege.getActions(), contains("data:write"));
+            assertThat(privilege.getMetadata().entrySet(), iterableWithSize(0));
+        }
+    }
+
+    private ApplicationPrivilegeDescriptor randomPrivilege() {
+        final String applicationName;
+        if (randomBoolean()) {
+            applicationName = "*";
+        } else {
+            applicationName = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(2, 10);
+        }
+        final String privilegeName = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(2, 8);
+        final String[] patterns = new String[randomIntBetween(0, 5)];
+        for (int i = 0; i < patterns.length; i++) {
+            final String suffix = randomBoolean() ? "*" : randomAlphaOfLengthBetween(4, 9);
+            patterns[i] = randomAlphaOfLengthBetween(2, 5) + "/" + suffix;
+        }
+
+        final Map<String, Object> metadata = new HashMap<>();
+        for (int i = randomInt(3); i > 0; i--) {
+            metadata.put(randomAlphaOfLengthBetween(2, 5), randomFrom(randomBoolean(), randomInt(10), randomAlphaOfLength(5)));
+        }
+        return new ApplicationPrivilegeDescriptor(applicationName, privilegeName, Sets.newHashSet(patterns), metadata);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeTests.java
new file mode 100644
index 00000000000..c65f06f05f9
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilegeTests.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import junit.framework.AssertionFailedError;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.junit.Assert;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ApplicationPrivilegeTests extends ESTestCase {
+
+    public void testValidationOfApplicationName() {
+        final String specialCharacters = ":;$#%()+='.{}[]!@^&'";
+        final Supplier<Character> specialCharacter = () -> specialCharacters.charAt(randomInt(specialCharacters.length() - 1));
+
+        assertValidationFailure("a p p", "application name", () -> ApplicationPrivilege.validateApplicationName("a p p"));
+        assertValidationFailure("ap", "application name", () -> ApplicationPrivilege.validateApplicationName("ap"));
+        for (String app : Arrays.asList(
+            "App",// must start with lowercase
+            "1app",  // must start with letter
+            "app" + specialCharacter.get() // cannot contain special characters unless preceded by a "-" or "_"
+        )) {
+            assertValidationFailure(app, "application name", () -> ApplicationPrivilege.validateApplicationName(app));
+            assertValidationFailure(app, "application name", () -> ApplicationPrivilege.validateApplicationNameOrWildcard(app));
+        }
+
+        // no wildcards
+        assertValidationFailure("app*", "application names", () -> ApplicationPrivilege.validateApplicationName("app*"));
+        // no special characters with wildcards
+        final String appNameWithSpecialCharAndWildcard = "app" + specialCharacter.get() + "*";
+        assertValidationFailure(appNameWithSpecialCharAndWildcard, "application name",
+            () -> ApplicationPrivilege.validateApplicationNameOrWildcard(appNameWithSpecialCharAndWildcard));
+
+        String appNameWithSpecialChars = "myapp" + randomFrom('-', '_');
+        for (int i = randomIntBetween(1, 12); i > 0; i--) {
+            appNameWithSpecialChars = appNameWithSpecialChars + specialCharacter.get();
+        }
+        // these should all be OK
+        for (String app : Arrays.asList("app", "app1", "myApp", "myApp-:;$#%()+='.", "myApp_:;$#%()+='.", appNameWithSpecialChars)) {
+            assertNoException(app, () -> ApplicationPrivilege.validateApplicationName(app));
+            assertNoException(app, () -> ApplicationPrivilege.validateApplicationNameOrWildcard(app));
+        }
+    }
+
+    public void testValidationOfPrivilegeName() {
+        // must start with lowercase
+        assertValidationFailure("Read", "privilege names", () -> ApplicationPrivilege.validatePrivilegeName("Read"));
+        // must start with letter
+        assertValidationFailure("1read", "privilege names", () -> ApplicationPrivilege.validatePrivilegeName("1read"));
+        // cannot contain special characters
+        final String specialChars = ":;$#%()+=/',";
+        final String withSpecialChar = "read" + specialChars.charAt(randomInt(specialChars.length()-1));
+        assertValidationFailure(withSpecialChar, "privilege names", () -> ApplicationPrivilege.validatePrivilegeName(withSpecialChar));
+
+        // these should all be OK
+        for (String priv : Arrays.asList("read", "read1", "readData", "read-data", "read.data", "read_data")) {
+            assertNoException(priv, () -> ApplicationPrivilege.validatePrivilegeName(priv));
+            assertNoException(priv, () -> ApplicationPrivilege.validatePrivilegeOrActionName(priv));
+        }
+
+        for (String priv : Arrays.asList("r e a d", "read\n", "copy®")) {
+            assertValidationFailure(priv, "privilege names and action", () -> ApplicationPrivilege.validatePrivilegeOrActionName(priv));
+        }
+
+        for (String priv : Arrays.asList("read:*", "read/*", "read/a_b.c-d+e%f#(g)")) {
+            assertNoException(priv, () -> ApplicationPrivilege.validatePrivilegeOrActionName(priv));
+        }
+    }
+
+    public void testNonePrivilege() {
+        final ApplicationPrivilege none = ApplicationPrivilege.NONE.apply("super-mega-app");
+        CharacterRunAutomaton run = new CharacterRunAutomaton(none.getAutomaton());
+        for (int i = randomIntBetween(5, 10); i > 0; i--) {
+            final String action;
+            if (randomBoolean()) {
+                action = randomAlphaOfLengthBetween(3, 12);
+            } else {
+                action = randomAlphaOfLengthBetween(3, 6) + randomFrom(":", "/") + randomAlphaOfLengthBetween(3, 8);
+            }
+            assertFalse("NONE should not grant " + action, run.run(action));
+        }
+    }
+
+    public void testGetPrivilegeByName() {
+        final ApplicationPrivilegeDescriptor descriptor = descriptor("my-app", "read", "data:read/*", "action:login");
+        final ApplicationPrivilegeDescriptor myWrite = descriptor("my-app", "write", "data:write/*", "action:login");
+        final ApplicationPrivilegeDescriptor myAdmin = descriptor("my-app", "admin", "data:read/*", "action:*");
+        final ApplicationPrivilegeDescriptor yourRead = descriptor("your-app", "read", "data:read/*", "action:login");
+        final Set<ApplicationPrivilegeDescriptor> stored = Sets.newHashSet(descriptor, myWrite, myAdmin, yourRead);
+
+        assertEqual(ApplicationPrivilege.get("my-app", Collections.singleton("read"), stored), descriptor);
+        assertEqual(ApplicationPrivilege.get("my-app", Collections.singleton("write"), stored), myWrite);
+
+        final ApplicationPrivilege readWrite = ApplicationPrivilege.get("my-app", Sets.newHashSet("read", "write"), stored);
+        assertThat(readWrite.getApplication(), equalTo("my-app"));
+        assertThat(readWrite.name(), containsInAnyOrder("read", "write"));
+        assertThat(readWrite.getPatterns(), arrayContainingInAnyOrder("data:read/*", "data:write/*", "action:login"));
+
+        CharacterRunAutomaton run = new CharacterRunAutomaton(readWrite.getAutomaton());
+        for (String action : Arrays.asList("data:read/settings", "data:write/user/kimchy", "action:login")) {
+            assertTrue(run.run(action));
+        }
+        for (String action : Arrays.asList("data:delete/user/kimchy", "action:shutdown")) {
+            assertFalse(run.run(action));
+        }
+    }
+
+    private void assertEqual(ApplicationPrivilege myReadPriv, ApplicationPrivilegeDescriptor myRead) {
+        assertThat(myReadPriv.getApplication(), equalTo(myRead.getApplication()));
+        assertThat(getPrivilegeName(myReadPriv), equalTo(myRead.getName()));
+        assertThat(Sets.newHashSet(myReadPriv.getPatterns()), equalTo(myRead.getActions()));
+    }
+
+    private ApplicationPrivilegeDescriptor descriptor(String application, String name, String... actions) {
+        return new ApplicationPrivilegeDescriptor(application, name, Sets.newHashSet(actions), Collections.emptyMap());
+    }
+
+    public void testEqualsAndHashCode() {
+        final ApplicationPrivilege privilege = randomPrivilege();
+        final EqualsHashCodeTestUtils.MutateFunction<ApplicationPrivilege> mutate = randomFrom(
+            orig -> createPrivilege("x" + orig.getApplication(), getPrivilegeName(orig), orig.getPatterns()),
+            orig -> createPrivilege(orig.getApplication(), "x" + getPrivilegeName(orig), orig.getPatterns()),
+            orig -> new ApplicationPrivilege(orig.getApplication(), getPrivilegeName(orig), "*")
+        );
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(privilege,
+            original -> createPrivilege(original.getApplication(), getPrivilegeName(original), original.getPatterns()),
+            mutate
+        );
+    }
+
+    private ApplicationPrivilege createPrivilege(String applicationName, String privilegeName, String... patterns) {
+        return new ApplicationPrivilege(applicationName, privilegeName, patterns);
+    }
+
+    private String getPrivilegeName(ApplicationPrivilege privilege) {
+        if (privilege.name.size() == 1) {
+            return privilege.name.iterator().next();
+        } else {
+            throw new IllegalStateException(privilege + " has a multivariate name: " + collectionToCommaDelimitedString(privilege.name));
+        }
+    }
+
+    private void assertValidationFailure(String reason,String messageContent, ThrowingRunnable body) {
+        final IllegalArgumentException exception;
+        try {
+            exception = expectThrows(IllegalArgumentException.class, body);
+            assertThat(exception.getMessage().toLowerCase(Locale.ROOT), containsString(messageContent.toLowerCase(Locale.ROOT)));
+        } catch (AssertionFailedError e) {
+            fail(reason + " - " + e.getMessage());
+        }
+    }
+
+    private void assertNoException(String reason, ThrowingRunnable body) {
+        try {
+            body.run();
+            // pass
+        } catch (Throwable e) {
+            Assert.fail(reason + " - Expected no exception, but got: " + e);
+        }
+    }
+
+    private ApplicationPrivilege randomPrivilege() {
+        final String applicationName;
+        if (randomBoolean()) {
+            applicationName = "*";
+        } else {
+            applicationName = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(2, 10);
+        }
+        final String privilegeName = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(2, 8);
+        final String[] patterns = new String[randomIntBetween(0, 5)];
+        for (int i = 0; i < patterns.length; i++) {
+            final String suffix = randomBoolean() ? "*" : randomAlphaOfLengthBetween(4, 9);
+            patterns[i] = randomAlphaOfLengthBetween(2, 5) + "/" + suffix;
+        }
+
+        final Map<String, Object> metadata = new HashMap<>();
+        for (int i = randomInt(3); i > 0; i--) {
+            metadata.put(randomAlphaOfLengthBetween(2, 5), randomFrom(randomBoolean(), randomInt(10), randomAlphaOfLength(5)));
+        }
+        return createPrivilege(applicationName, privilegeName, patterns);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivilegesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivilegesTests.java
new file mode 100644
index 00000000000..ebcd70869cb
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivilegesTests.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ConditionalClusterPrivilegesTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        final ConditionalClusterPrivilege[] original = buildSecurityPrivileges();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            ConditionalClusterPrivileges.writeArray(out, original);
+            final NamedWriteableRegistry registry = new NamedWriteableRegistry(new XPackClientPlugin(Settings.EMPTY).getNamedWriteables());
+            try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry)) {
+                final ConditionalClusterPrivilege[] copy = ConditionalClusterPrivileges.readArray(in);
+                assertThat(copy, equalTo(original));
+                assertThat(original, equalTo(copy));
+            }
+        }
+    }
+
+    public void testGenerateAndParseXContent() throws Exception {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            final XContentBuilder builder = new XContentBuilder(xContent, out);
+
+            final List<ConditionalClusterPrivilege> original = Arrays.asList(buildSecurityPrivileges());
+            ConditionalClusterPrivileges.toXContent(builder, ToXContent.EMPTY_PARAMS, original);
+            builder.flush();
+
+            final byte[] bytes = out.toByteArray();
+            try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, bytes)) {
+                assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
+                final List<ConditionalClusterPrivilege> clone = ConditionalClusterPrivileges.parse(parser);
+                assertThat(clone, equalTo(original));
+                assertThat(original, equalTo(clone));
+            }
+        }
+    }
+
+    private ConditionalClusterPrivilege[] buildSecurityPrivileges() {
+        return buildSecurityPrivileges(randomIntBetween(4, 7));
+    }
+
+    private ConditionalClusterPrivilege[] buildSecurityPrivileges(int applicationNameLength) {
+        return new ConditionalClusterPrivilege[] {
+            ManageApplicationPrivilegesTests.buildPrivileges(applicationNameLength)
+        };
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
new file mode 100644
index 00000000000..a5c1bbc98d1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.role.PutRoleAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
+import org.elasticsearch.xpack.core.security.action.user.GetUsersAction;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges.ManageApplicationPrivileges;
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION;
+import static org.elasticsearch.test.TestMatchers.predicateMatches;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class ManageApplicationPrivilegesTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        final ManageApplicationPrivileges original = buildPrivileges();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            original.writeTo(out);
+            final NamedWriteableRegistry registry = new NamedWriteableRegistry(new XPackClientPlugin(Settings.EMPTY).getNamedWriteables());
+            try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry)) {
+                final ManageApplicationPrivileges copy = ManageApplicationPrivileges.createFrom(in);
+                assertThat(copy, equalTo(original));
+                assertThat(original, equalTo(copy));
+            }
+        }
+    }
+
+    public void testGenerateAndParseXContent() throws Exception {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            final XContentBuilder builder = new XContentBuilder(xContent, out);
+
+            final ManageApplicationPrivileges original = buildPrivileges();
+            builder.startObject();
+            original.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            builder.flush();
+
+            final byte[] bytes = out.toByteArray();
+            try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, bytes)) {
+                assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
+                // ManageApplicationPrivileges.parse requires that the parser be positioned on the "manage" field.
+                assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
+                final ManageApplicationPrivileges clone = ManageApplicationPrivileges.parse(parser);
+                assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
+
+                assertThat(clone, equalTo(original));
+                assertThat(original, equalTo(clone));
+            }
+        }
+    }
+
+    public void testEqualsAndHashCode() {
+        final int applicationNameLength = randomIntBetween(4, 7);
+        final ManageApplicationPrivileges privileges = buildPrivileges(applicationNameLength);
+        final EqualsHashCodeTestUtils.MutateFunction<ManageApplicationPrivileges> mutate
+            = orig -> buildPrivileges(applicationNameLength + randomIntBetween(1, 3));
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(privileges, this::clone, mutate);
+    }
+
+    public void testPrivilege() {
+        final ManageApplicationPrivileges privileges = buildPrivileges();
+        assertThat(privileges.getPrivilege(), instanceOf(ClusterPrivilege.class));
+        for (String actionName : Arrays.asList(GetPrivilegesAction.NAME, PutPrivilegesAction.NAME, DeletePrivilegesAction.NAME)) {
+            assertThat(privileges.getPrivilege().predicate(), predicateMatches(actionName));
+        }
+        for (String actionName : Arrays.asList(GetUsersAction.NAME, PutRoleAction.NAME, DeleteRoleMappingAction.NAME,
+            HasPrivilegesAction.NAME)) {
+            assertThat(privileges.getPrivilege().predicate(), not(predicateMatches(actionName)));
+        }
+    }
+
+    public void testRequestPredicate() {
+        final ManageApplicationPrivileges kibanaAndLogstash = new ManageApplicationPrivileges(Sets.newHashSet("kibana-*", "logstash"));
+        final ManageApplicationPrivileges cloudAndSwiftype = new ManageApplicationPrivileges(Sets.newHashSet("cloud-*", "swiftype"));
+        final Predicate<TransportRequest> kibanaAndLogstashPredicate = kibanaAndLogstash.getRequestPredicate();
+        final Predicate<TransportRequest> cloudAndSwiftypePredicate = cloudAndSwiftype.getRequestPredicate();
+        assertThat(kibanaAndLogstashPredicate, notNullValue());
+        assertThat(cloudAndSwiftypePredicate, notNullValue());
+
+        final GetPrivilegesRequest getKibana1 = new GetPrivilegesRequest();
+        getKibana1.application("kibana-1");
+        assertThat(kibanaAndLogstashPredicate, predicateMatches(getKibana1));
+        assertThat(cloudAndSwiftypePredicate, not(predicateMatches(getKibana1)));
+
+        final DeletePrivilegesRequest deleteLogstash = new DeletePrivilegesRequest("logstash", new String[]{"all"});
+        assertThat(kibanaAndLogstashPredicate, predicateMatches(deleteLogstash));
+        assertThat(cloudAndSwiftypePredicate, not(predicateMatches(deleteLogstash)));
+
+        final PutPrivilegesRequest putKibana = new PutPrivilegesRequest();
+
+        final List<ApplicationPrivilegeDescriptor> kibanaPrivileges = new ArrayList<>();
+        for (int i = randomIntBetween(2, 6); i > 0; i--) {
+            kibanaPrivileges.add(new ApplicationPrivilegeDescriptor("kibana-" + i,
+                randomAlphaOfLengthBetween(3, 6).toLowerCase(Locale.ROOT), Collections.emptySet(), Collections.emptyMap()));
+        }
+        putKibana.setPrivileges(kibanaPrivileges);
+        assertThat(kibanaAndLogstashPredicate, predicateMatches(putKibana));
+        assertThat(cloudAndSwiftypePredicate, not(predicateMatches(putKibana)));
+    }
+
+
+    private ManageApplicationPrivileges clone(ManageApplicationPrivileges original) {
+        return new ManageApplicationPrivileges(new LinkedHashSet<>(original.getApplicationNames()));
+    }
+
+    private ManageApplicationPrivileges buildPrivileges() {
+        return buildPrivileges(randomIntBetween(4, 7));
+    }
+
+    static ManageApplicationPrivileges buildPrivileges(int applicationNameLength) {
+        Set<String> applicationNames = Sets.newHashSet(Arrays.asList(generateRandomStringArray(5, applicationNameLength, false, false)));
+        return new ManageApplicationPrivileges(applicationNames);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index 85d2bc16dd0..9cb5e25c5b8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
@@ -75,6 +76,12 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.role.PutRoleAction;
 import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.saml.SamlPrepareAuthenticationAction;
@@ -85,6 +92,8 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl.IndexAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
@@ -104,10 +113,12 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
 
 /**
  * Unit tests for the {@link ReservedRolesStore}
@@ -139,21 +150,23 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testIngestAdminRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("ingest_admin");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role ingestAdminRole = Role.builder(roleDescriptor, null).build();
-        assertThat(ingestAdminRole.cluster().check(PutIndexTemplateAction.NAME), is(true));
-        assertThat(ingestAdminRole.cluster().check(GetIndexTemplatesAction.NAME), is(true));
-        assertThat(ingestAdminRole.cluster().check(DeleteIndexTemplateAction.NAME), is(true));
-        assertThat(ingestAdminRole.cluster().check(PutPipelineAction.NAME), is(true));
-        assertThat(ingestAdminRole.cluster().check(GetPipelineAction.NAME), is(true));
-        assertThat(ingestAdminRole.cluster().check(DeletePipelineAction.NAME), is(true));
+        assertThat(ingestAdminRole.cluster().check(PutIndexTemplateAction.NAME, request), is(true));
+        assertThat(ingestAdminRole.cluster().check(GetIndexTemplatesAction.NAME, request), is(true));
+        assertThat(ingestAdminRole.cluster().check(DeleteIndexTemplateAction.NAME, request), is(true));
+        assertThat(ingestAdminRole.cluster().check(PutPipelineAction.NAME, request), is(true));
+        assertThat(ingestAdminRole.cluster().check(GetPipelineAction.NAME, request), is(true));
+        assertThat(ingestAdminRole.cluster().check(DeletePipelineAction.NAME, request), is(true));
 
-        assertThat(ingestAdminRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(ingestAdminRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(ingestAdminRole.cluster().check(MonitoringBulkAction.NAME), is(false));
+        assertThat(ingestAdminRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(ingestAdminRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(ingestAdminRole.cluster().check(MonitoringBulkAction.NAME, request), is(false));
 
         assertThat(ingestAdminRole.indices().allowedIndicesMatcher(IndexAction.NAME).test("foo"), is(false));
         assertThat(ingestAdminRole.indices().allowedIndicesMatcher("indices:foo").test(randomAlphaOfLengthBetween(8, 24)),
@@ -163,25 +176,49 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testKibanaSystemRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("kibana_system");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role kibanaRole = Role.builder(roleDescriptor, null).build();
-        assertThat(kibanaRole.cluster().check(ClusterHealthAction.NAME), is(true));
-        assertThat(kibanaRole.cluster().check(ClusterStateAction.NAME), is(true));
-        assertThat(kibanaRole.cluster().check(ClusterStatsAction.NAME), is(true));
-        assertThat(kibanaRole.cluster().check(PutIndexTemplateAction.NAME), is(true));
-        assertThat(kibanaRole.cluster().check(GetIndexTemplatesAction.NAME), is(true));
-        assertThat(kibanaRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(kibanaRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(kibanaRole.cluster().check(MonitoringBulkAction.NAME), is(true));
+        assertThat(kibanaRole.cluster().check(ClusterHealthAction.NAME, request), is(true));
+        assertThat(kibanaRole.cluster().check(ClusterStateAction.NAME, request), is(true));
+        assertThat(kibanaRole.cluster().check(ClusterStatsAction.NAME, request), is(true));
+        assertThat(kibanaRole.cluster().check(PutIndexTemplateAction.NAME, request), is(true));
+        assertThat(kibanaRole.cluster().check(GetIndexTemplatesAction.NAME, request), is(true));
+        assertThat(kibanaRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(kibanaRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(kibanaRole.cluster().check(MonitoringBulkAction.NAME, request), is(true));
 
         // SAML
-        assertThat(kibanaRole.cluster().check(SamlPrepareAuthenticationAction.NAME), is(true));
-        assertThat(kibanaRole.cluster().check(SamlAuthenticateAction.NAME), is(true));
-        assertThat(kibanaRole.cluster().check(InvalidateTokenAction.NAME), is(true));
-        assertThat(kibanaRole.cluster().check(CreateTokenAction.NAME), is(false));
+        assertThat(kibanaRole.cluster().check(SamlPrepareAuthenticationAction.NAME, request), is(true));
+        assertThat(kibanaRole.cluster().check(SamlAuthenticateAction.NAME, request), is(true));
+        assertThat(kibanaRole.cluster().check(InvalidateTokenAction.NAME, request), is(true));
+        assertThat(kibanaRole.cluster().check(CreateTokenAction.NAME, request), is(false));
+
+        // Application Privileges
+        DeletePrivilegesRequest deleteKibanaPrivileges = new DeletePrivilegesRequest("kibana-.kibana", new String[]{ "all", "read" });
+        DeletePrivilegesRequest deleteLogstashPrivileges = new DeletePrivilegesRequest("logstash", new String[]{ "all", "read" });
+        assertThat(kibanaRole.cluster().check(DeletePrivilegesAction.NAME, deleteKibanaPrivileges), is(true));
+        assertThat(kibanaRole.cluster().check(DeletePrivilegesAction.NAME, deleteLogstashPrivileges), is(false));
+
+        GetPrivilegesRequest getKibanaPrivileges = new GetPrivilegesRequest();
+        getKibanaPrivileges.application("kibana-.kibana-sales");
+        GetPrivilegesRequest getApmPrivileges = new GetPrivilegesRequest();
+        getApmPrivileges.application("apm");
+        assertThat(kibanaRole.cluster().check(GetPrivilegesAction.NAME, getKibanaPrivileges), is(true));
+        assertThat(kibanaRole.cluster().check(GetPrivilegesAction.NAME, getApmPrivileges), is(false));
+
+        PutPrivilegesRequest putKibanaPrivileges = new PutPrivilegesRequest();
+        putKibanaPrivileges.setPrivileges(Collections.singletonList(new ApplicationPrivilegeDescriptor(
+            "kibana-.kibana-" + randomAlphaOfLengthBetween(2,6), "all", Collections.emptySet(), Collections.emptyMap())));
+        PutPrivilegesRequest putSwiftypePrivileges = new PutPrivilegesRequest();
+        putSwiftypePrivileges.setPrivileges(Collections.singletonList(new ApplicationPrivilegeDescriptor(
+            "swiftype-kibana" , "all", Collections.emptySet(), Collections.emptyMap())));
+        assertThat(kibanaRole.cluster().check(PutPrivilegesAction.NAME, putKibanaPrivileges), is(true));
+        assertThat(kibanaRole.cluster().check(PutPrivilegesAction.NAME, putSwiftypePrivileges), is(false));
 
         // Everything else
         assertThat(kibanaRole.runAs().check(randomAlphaOfLengthBetween(1, 12)), is(false));
@@ -238,18 +275,20 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testKibanaUserRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("kibana_user");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role kibanaUserRole = Role.builder(roleDescriptor, null).build();
-        assertThat(kibanaUserRole.cluster().check(ClusterHealthAction.NAME), is(false));
-        assertThat(kibanaUserRole.cluster().check(ClusterStateAction.NAME), is(false));
-        assertThat(kibanaUserRole.cluster().check(ClusterStatsAction.NAME), is(false));
-        assertThat(kibanaUserRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
-        assertThat(kibanaUserRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(kibanaUserRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(kibanaUserRole.cluster().check(MonitoringBulkAction.NAME), is(false));
+        assertThat(kibanaUserRole.cluster().check(ClusterHealthAction.NAME, request), is(false));
+        assertThat(kibanaUserRole.cluster().check(ClusterStateAction.NAME, request), is(false));
+        assertThat(kibanaUserRole.cluster().check(ClusterStatsAction.NAME, request), is(false));
+        assertThat(kibanaUserRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(kibanaUserRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(kibanaUserRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(kibanaUserRole.cluster().check(MonitoringBulkAction.NAME, request), is(false));
 
         assertThat(kibanaUserRole.runAs().check(randomAlphaOfLengthBetween(1, 12)), is(false));
 
@@ -271,22 +310,35 @@ public class ReservedRolesStoreTests extends ESTestCase {
             assertThat(kibanaUserRole.indices().allowedIndicesMatcher(MultiSearchAction.NAME).test(index), is(true));
             assertThat(kibanaUserRole.indices().allowedIndicesMatcher(UpdateSettingsAction.NAME).test(index), is(true));
         });
+
+        final String randomApplication = "kibana-" + randomAlphaOfLengthBetween(8, 24);
+        assertThat(kibanaUserRole.application().grants(new ApplicationPrivilege(randomApplication, "app-random", "all"), "*"), is(false));
+
+        final String application = "kibana-.kibana";
+        assertThat(kibanaUserRole.application().grants(new ApplicationPrivilege(application, "app-foo", "foo"), "*"), is(false));
+        assertThat(kibanaUserRole.application().grants(new ApplicationPrivilege(application, "app-all", "all"), "*"), is(true));
+
+        final String applicationWithRandomIndex = "kibana-.kibana_" + randomAlphaOfLengthBetween(8, 24);
+        assertThat(kibanaUserRole.application().grants(new ApplicationPrivilege(applicationWithRandomIndex, "app-random-index", "all"),
+            "*"), is(false));
     }
 
     public void testMonitoringUserRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("monitoring_user");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role monitoringUserRole = Role.builder(roleDescriptor, null).build();
-        assertThat(monitoringUserRole.cluster().check(MainAction.NAME), is(true));
-        assertThat(monitoringUserRole.cluster().check(ClusterHealthAction.NAME), is(false));
-        assertThat(monitoringUserRole.cluster().check(ClusterStateAction.NAME), is(false));
-        assertThat(monitoringUserRole.cluster().check(ClusterStatsAction.NAME), is(false));
-        assertThat(monitoringUserRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
-        assertThat(monitoringUserRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(monitoringUserRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(monitoringUserRole.cluster().check(MonitoringBulkAction.NAME), is(false));
+        assertThat(monitoringUserRole.cluster().check(MainAction.NAME, request), is(true));
+        assertThat(monitoringUserRole.cluster().check(ClusterHealthAction.NAME, request), is(false));
+        assertThat(monitoringUserRole.cluster().check(ClusterStateAction.NAME, request), is(false));
+        assertThat(monitoringUserRole.cluster().check(ClusterStatsAction.NAME, request), is(false));
+        assertThat(monitoringUserRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(monitoringUserRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(monitoringUserRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(monitoringUserRole.cluster().check(MonitoringBulkAction.NAME, request), is(false));
 
         assertThat(monitoringUserRole.runAs().check(randomAlphaOfLengthBetween(1, 12)), is(false));
 
@@ -313,27 +365,29 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testRemoteMonitoringAgentRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("remote_monitoring_agent");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role remoteMonitoringAgentRole = Role.builder(roleDescriptor, null).build();
-        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterHealthAction.NAME), is(true));
-        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterStateAction.NAME), is(true));
-        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterStatsAction.NAME), is(true));
-        assertThat(remoteMonitoringAgentRole.cluster().check(PutIndexTemplateAction.NAME), is(true));
-        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(remoteMonitoringAgentRole.cluster().check(MonitoringBulkAction.NAME), is(false));
-        assertThat(remoteMonitoringAgentRole.cluster().check(GetWatchAction.NAME), is(true));
-        assertThat(remoteMonitoringAgentRole.cluster().check(PutWatchAction.NAME), is(true));
-        assertThat(remoteMonitoringAgentRole.cluster().check(DeleteWatchAction.NAME), is(true));
-        assertThat(remoteMonitoringAgentRole.cluster().check(ExecuteWatchAction.NAME), is(false));
-        assertThat(remoteMonitoringAgentRole.cluster().check(AckWatchAction.NAME), is(false));
-        assertThat(remoteMonitoringAgentRole.cluster().check(ActivateWatchAction.NAME), is(false));
-        assertThat(remoteMonitoringAgentRole.cluster().check(WatcherServiceAction.NAME), is(false));
+        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterHealthAction.NAME, request), is(true));
+        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterStateAction.NAME, request), is(true));
+        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterStatsAction.NAME, request), is(true));
+        assertThat(remoteMonitoringAgentRole.cluster().check(PutIndexTemplateAction.NAME, request), is(true));
+        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(remoteMonitoringAgentRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(remoteMonitoringAgentRole.cluster().check(MonitoringBulkAction.NAME, request), is(false));
+        assertThat(remoteMonitoringAgentRole.cluster().check(GetWatchAction.NAME, request), is(true));
+        assertThat(remoteMonitoringAgentRole.cluster().check(PutWatchAction.NAME, request), is(true));
+        assertThat(remoteMonitoringAgentRole.cluster().check(DeleteWatchAction.NAME, request), is(true));
+        assertThat(remoteMonitoringAgentRole.cluster().check(ExecuteWatchAction.NAME, request), is(false));
+        assertThat(remoteMonitoringAgentRole.cluster().check(AckWatchAction.NAME, request), is(false));
+        assertThat(remoteMonitoringAgentRole.cluster().check(ActivateWatchAction.NAME, request), is(false));
+        assertThat(remoteMonitoringAgentRole.cluster().check(WatcherServiceAction.NAME, request), is(false));
         // we get this from the cluster:monitor privilege
-        assertThat(remoteMonitoringAgentRole.cluster().check(WatcherStatsAction.NAME), is(true));
+        assertThat(remoteMonitoringAgentRole.cluster().check(WatcherStatsAction.NAME, request), is(true));
 
         assertThat(remoteMonitoringAgentRole.runAs().check(randomAlphaOfLengthBetween(1, 12)), is(false));
 
@@ -357,18 +411,20 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testReportingUserRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("reporting_user");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role reportingUserRole = Role.builder(roleDescriptor, null).build();
-        assertThat(reportingUserRole.cluster().check(ClusterHealthAction.NAME), is(false));
-        assertThat(reportingUserRole.cluster().check(ClusterStateAction.NAME), is(false));
-        assertThat(reportingUserRole.cluster().check(ClusterStatsAction.NAME), is(false));
-        assertThat(reportingUserRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
-        assertThat(reportingUserRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(reportingUserRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(reportingUserRole.cluster().check(MonitoringBulkAction.NAME), is(false));
+        assertThat(reportingUserRole.cluster().check(ClusterHealthAction.NAME, request), is(false));
+        assertThat(reportingUserRole.cluster().check(ClusterStateAction.NAME, request), is(false));
+        assertThat(reportingUserRole.cluster().check(ClusterStatsAction.NAME, request), is(false));
+        assertThat(reportingUserRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(reportingUserRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(reportingUserRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(reportingUserRole.cluster().check(MonitoringBulkAction.NAME, request), is(false));
 
         assertThat(reportingUserRole.runAs().check(randomAlphaOfLengthBetween(1, 12)), is(false));
 
@@ -393,18 +449,20 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testKibanaDashboardOnlyUserRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("kibana_dashboard_only_user");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role dashboardsOnlyUserRole = Role.builder(roleDescriptor, null).build();
-        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterHealthAction.NAME), is(false));
-        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterStateAction.NAME), is(false));
-        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterStatsAction.NAME), is(false));
-        assertThat(dashboardsOnlyUserRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
-        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(dashboardsOnlyUserRole.cluster().check(MonitoringBulkAction.NAME), is(false));
+        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterHealthAction.NAME, request), is(false));
+        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterStateAction.NAME, request), is(false));
+        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterStatsAction.NAME, request), is(false));
+        assertThat(dashboardsOnlyUserRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(dashboardsOnlyUserRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(dashboardsOnlyUserRole.cluster().check(MonitoringBulkAction.NAME, request), is(false));
 
         assertThat(dashboardsOnlyUserRole.runAs().check(randomAlphaOfLengthBetween(1, 12)), is(false));
 
@@ -420,20 +478,35 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(dashboardsOnlyUserRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(index), is(true));
         assertThat(dashboardsOnlyUserRole.indices().allowedIndicesMatcher(SearchAction.NAME).test(index), is(true));
         assertThat(dashboardsOnlyUserRole.indices().allowedIndicesMatcher(MultiSearchAction.NAME).test(index), is(true));
+
+        final String randomApplication = "kibana-" + randomAlphaOfLengthBetween(8, 24);
+        assertThat(dashboardsOnlyUserRole.application().grants(new ApplicationPrivilege(randomApplication, "app-random", "all"), "*"),
+            is(false));
+
+        final String application = "kibana-.kibana";
+        assertThat(dashboardsOnlyUserRole.application().grants(new ApplicationPrivilege(application, "app-foo", "foo"), "*"), is(false));
+        assertThat(dashboardsOnlyUserRole.application().grants(new ApplicationPrivilege(application, "app-all", "all"), "*"), is(false));
+        assertThat(dashboardsOnlyUserRole.application().grants(new ApplicationPrivilege(application, "app-read", "read"), "*"), is(true));
+
+        final String applicationWithRandomIndex = "kibana-.kibana_" + randomAlphaOfLengthBetween(8, 24);
+        assertThat(dashboardsOnlyUserRole.application().grants(
+            new ApplicationPrivilege(applicationWithRandomIndex, "app-random-index", "all"), "*"), is(false));
     }
 
     public void testSuperuserRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("superuser");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role superuserRole = Role.builder(roleDescriptor, null).build();
-        assertThat(superuserRole.cluster().check(ClusterHealthAction.NAME), is(true));
-        assertThat(superuserRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(true));
-        assertThat(superuserRole.cluster().check(PutUserAction.NAME), is(true));
-        assertThat(superuserRole.cluster().check(PutRoleAction.NAME), is(true));
-        assertThat(superuserRole.cluster().check(PutIndexTemplateAction.NAME), is(true));
-        assertThat(superuserRole.cluster().check("internal:admin/foo"), is(false));
+        assertThat(superuserRole.cluster().check(ClusterHealthAction.NAME, request), is(true));
+        assertThat(superuserRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(true));
+        assertThat(superuserRole.cluster().check(PutUserAction.NAME, request), is(true));
+        assertThat(superuserRole.cluster().check(PutRoleAction.NAME, request), is(true));
+        assertThat(superuserRole.cluster().check(PutIndexTemplateAction.NAME, request), is(true));
+        assertThat(superuserRole.cluster().check("internal:admin/foo", request), is(false));
 
         final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
         final MetaData metaData = new MetaData.Builder()
@@ -472,18 +545,20 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testLogstashSystemRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("logstash_system");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role logstashSystemRole = Role.builder(roleDescriptor, null).build();
-        assertThat(logstashSystemRole.cluster().check(ClusterHealthAction.NAME), is(true));
-        assertThat(logstashSystemRole.cluster().check(ClusterStateAction.NAME), is(true));
-        assertThat(logstashSystemRole.cluster().check(ClusterStatsAction.NAME), is(true));
-        assertThat(logstashSystemRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
-        assertThat(logstashSystemRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(logstashSystemRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(logstashSystemRole.cluster().check(MonitoringBulkAction.NAME), is(true));
+        assertThat(logstashSystemRole.cluster().check(ClusterHealthAction.NAME, request), is(true));
+        assertThat(logstashSystemRole.cluster().check(ClusterStateAction.NAME, request), is(true));
+        assertThat(logstashSystemRole.cluster().check(ClusterStatsAction.NAME, request), is(true));
+        assertThat(logstashSystemRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(logstashSystemRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(logstashSystemRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(logstashSystemRole.cluster().check(MonitoringBulkAction.NAME, request), is(true));
 
         assertThat(logstashSystemRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
@@ -494,18 +569,21 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testBeatsAdminRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         final RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("beats_admin");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
+
         final Role beatsAdminRole = Role.builder(roleDescriptor, null).build();
-        assertThat(beatsAdminRole.cluster().check(ClusterHealthAction.NAME), is(false));
-        assertThat(beatsAdminRole.cluster().check(ClusterStateAction.NAME), is(false));
-        assertThat(beatsAdminRole.cluster().check(ClusterStatsAction.NAME), is(false));
-        assertThat(beatsAdminRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
-        assertThat(beatsAdminRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(beatsAdminRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(beatsAdminRole.cluster().check(MonitoringBulkAction.NAME), is(false));
+        assertThat(beatsAdminRole.cluster().check(ClusterHealthAction.NAME, request), is(false));
+        assertThat(beatsAdminRole.cluster().check(ClusterStateAction.NAME, request), is(false));
+        assertThat(beatsAdminRole.cluster().check(ClusterStatsAction.NAME, request), is(false));
+        assertThat(beatsAdminRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(beatsAdminRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(beatsAdminRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(beatsAdminRole.cluster().check(MonitoringBulkAction.NAME, request), is(false));
 
         assertThat(beatsAdminRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
@@ -527,18 +605,20 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testBeatsSystemRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor(BeatsSystemUser.ROLE_NAME);
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role logstashSystemRole = Role.builder(roleDescriptor, null).build();
-        assertThat(logstashSystemRole.cluster().check(ClusterHealthAction.NAME), is(true));
-        assertThat(logstashSystemRole.cluster().check(ClusterStateAction.NAME), is(true));
-        assertThat(logstashSystemRole.cluster().check(ClusterStatsAction.NAME), is(true));
-        assertThat(logstashSystemRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
-        assertThat(logstashSystemRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(logstashSystemRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
-        assertThat(logstashSystemRole.cluster().check(MonitoringBulkAction.NAME), is(true));
+        assertThat(logstashSystemRole.cluster().check(ClusterHealthAction.NAME, request), is(true));
+        assertThat(logstashSystemRole.cluster().check(ClusterStateAction.NAME, request), is(true));
+        assertThat(logstashSystemRole.cluster().check(ClusterStatsAction.NAME, request), is(true));
+        assertThat(logstashSystemRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(logstashSystemRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(logstashSystemRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
+        assertThat(logstashSystemRole.cluster().check(MonitoringBulkAction.NAME, request), is(true));
 
         assertThat(logstashSystemRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
@@ -549,46 +629,48 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testMachineLearningAdminRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("machine_learning_admin");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role role = Role.builder(roleDescriptor, null).build();
-        assertThat(role.cluster().check(CloseJobAction.NAME), is(true));
-        assertThat(role.cluster().check(DeleteDatafeedAction.NAME), is(true));
-        assertThat(role.cluster().check(DeleteExpiredDataAction.NAME), is(true));
-        assertThat(role.cluster().check(DeleteFilterAction.NAME), is(true));
-        assertThat(role.cluster().check(DeleteJobAction.NAME), is(true));
-        assertThat(role.cluster().check(DeleteModelSnapshotAction.NAME), is(true));
-        assertThat(role.cluster().check(FinalizeJobExecutionAction.NAME), is(false)); // internal use only
-        assertThat(role.cluster().check(FlushJobAction.NAME), is(true));
-        assertThat(role.cluster().check(GetBucketsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetCategoriesAction.NAME), is(true));
-        assertThat(role.cluster().check(GetDatafeedsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetDatafeedsStatsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetFiltersAction.NAME), is(true));
-        assertThat(role.cluster().check(GetInfluencersAction.NAME), is(true));
-        assertThat(role.cluster().check(GetJobsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetJobsStatsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetModelSnapshotsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetRecordsAction.NAME), is(true));
-        assertThat(role.cluster().check(IsolateDatafeedAction.NAME), is(false)); // internal use only
-        assertThat(role.cluster().check(KillProcessAction.NAME), is(false)); // internal use only
-        assertThat(role.cluster().check(OpenJobAction.NAME), is(true));
-        assertThat(role.cluster().check(PostDataAction.NAME), is(true));
-        assertThat(role.cluster().check(PreviewDatafeedAction.NAME), is(true));
-        assertThat(role.cluster().check(PutDatafeedAction.NAME), is(true));
-        assertThat(role.cluster().check(PutFilterAction.NAME), is(true));
-        assertThat(role.cluster().check(PutJobAction.NAME), is(true));
-        assertThat(role.cluster().check(RevertModelSnapshotAction.NAME), is(true));
-        assertThat(role.cluster().check(StartDatafeedAction.NAME), is(true));
-        assertThat(role.cluster().check(StopDatafeedAction.NAME), is(true));
-        assertThat(role.cluster().check(UpdateDatafeedAction.NAME), is(true));
-        assertThat(role.cluster().check(UpdateJobAction.NAME), is(true));
-        assertThat(role.cluster().check(UpdateModelSnapshotAction.NAME), is(true));
-        assertThat(role.cluster().check(UpdateProcessAction.NAME), is(false)); // internal use only
-        assertThat(role.cluster().check(ValidateDetectorAction.NAME), is(true));
-        assertThat(role.cluster().check(ValidateJobConfigAction.NAME), is(true));
+        assertThat(role.cluster().check(CloseJobAction.NAME, request), is(true));
+        assertThat(role.cluster().check(DeleteDatafeedAction.NAME, request), is(true));
+        assertThat(role.cluster().check(DeleteExpiredDataAction.NAME, request), is(true));
+        assertThat(role.cluster().check(DeleteFilterAction.NAME, request), is(true));
+        assertThat(role.cluster().check(DeleteJobAction.NAME, request), is(true));
+        assertThat(role.cluster().check(DeleteModelSnapshotAction.NAME, request), is(true));
+        assertThat(role.cluster().check(FinalizeJobExecutionAction.NAME, request), is(false)); // internal use only
+        assertThat(role.cluster().check(FlushJobAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetBucketsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetCategoriesAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetDatafeedsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetDatafeedsStatsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetFiltersAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetInfluencersAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetJobsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetJobsStatsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetModelSnapshotsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetRecordsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(IsolateDatafeedAction.NAME, request), is(false)); // internal use only
+        assertThat(role.cluster().check(KillProcessAction.NAME, request), is(false)); // internal use only
+        assertThat(role.cluster().check(OpenJobAction.NAME, request), is(true));
+        assertThat(role.cluster().check(PostDataAction.NAME, request), is(true));
+        assertThat(role.cluster().check(PreviewDatafeedAction.NAME, request), is(true));
+        assertThat(role.cluster().check(PutDatafeedAction.NAME, request), is(true));
+        assertThat(role.cluster().check(PutFilterAction.NAME, request), is(true));
+        assertThat(role.cluster().check(PutJobAction.NAME, request), is(true));
+        assertThat(role.cluster().check(RevertModelSnapshotAction.NAME, request), is(true));
+        assertThat(role.cluster().check(StartDatafeedAction.NAME, request), is(true));
+        assertThat(role.cluster().check(StopDatafeedAction.NAME, request), is(true));
+        assertThat(role.cluster().check(UpdateDatafeedAction.NAME, request), is(true));
+        assertThat(role.cluster().check(UpdateJobAction.NAME, request), is(true));
+        assertThat(role.cluster().check(UpdateModelSnapshotAction.NAME, request), is(true));
+        assertThat(role.cluster().check(UpdateProcessAction.NAME, request), is(false)); // internal use only
+        assertThat(role.cluster().check(ValidateDetectorAction.NAME, request), is(true));
+        assertThat(role.cluster().check(ValidateJobConfigAction.NAME, request), is(true));
         assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
         assertNoAccessAllowed(role, "foo");
@@ -599,46 +681,48 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testMachineLearningUserRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("machine_learning_user");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role role = Role.builder(roleDescriptor, null).build();
-        assertThat(role.cluster().check(CloseJobAction.NAME), is(false));
-        assertThat(role.cluster().check(DeleteDatafeedAction.NAME), is(false));
-        assertThat(role.cluster().check(DeleteExpiredDataAction.NAME), is(false));
-        assertThat(role.cluster().check(DeleteFilterAction.NAME), is(false));
-        assertThat(role.cluster().check(DeleteJobAction.NAME), is(false));
-        assertThat(role.cluster().check(DeleteModelSnapshotAction.NAME), is(false));
-        assertThat(role.cluster().check(FinalizeJobExecutionAction.NAME), is(false));
-        assertThat(role.cluster().check(FlushJobAction.NAME), is(false));
-        assertThat(role.cluster().check(GetBucketsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetCategoriesAction.NAME), is(true));
-        assertThat(role.cluster().check(GetDatafeedsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetDatafeedsStatsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetFiltersAction.NAME), is(false));
-        assertThat(role.cluster().check(GetInfluencersAction.NAME), is(true));
-        assertThat(role.cluster().check(GetJobsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetJobsStatsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetModelSnapshotsAction.NAME), is(true));
-        assertThat(role.cluster().check(GetRecordsAction.NAME), is(true));
-        assertThat(role.cluster().check(IsolateDatafeedAction.NAME), is(false));
-        assertThat(role.cluster().check(KillProcessAction.NAME), is(false));
-        assertThat(role.cluster().check(OpenJobAction.NAME), is(false));
-        assertThat(role.cluster().check(PostDataAction.NAME), is(false));
-        assertThat(role.cluster().check(PreviewDatafeedAction.NAME), is(false));
-        assertThat(role.cluster().check(PutDatafeedAction.NAME), is(false));
-        assertThat(role.cluster().check(PutFilterAction.NAME), is(false));
-        assertThat(role.cluster().check(PutJobAction.NAME), is(false));
-        assertThat(role.cluster().check(RevertModelSnapshotAction.NAME), is(false));
-        assertThat(role.cluster().check(StartDatafeedAction.NAME), is(false));
-        assertThat(role.cluster().check(StopDatafeedAction.NAME), is(false));
-        assertThat(role.cluster().check(UpdateDatafeedAction.NAME), is(false));
-        assertThat(role.cluster().check(UpdateJobAction.NAME), is(false));
-        assertThat(role.cluster().check(UpdateModelSnapshotAction.NAME), is(false));
-        assertThat(role.cluster().check(UpdateProcessAction.NAME), is(false));
-        assertThat(role.cluster().check(ValidateDetectorAction.NAME), is(false));
-        assertThat(role.cluster().check(ValidateJobConfigAction.NAME), is(false));
+        assertThat(role.cluster().check(CloseJobAction.NAME, request), is(false));
+        assertThat(role.cluster().check(DeleteDatafeedAction.NAME, request), is(false));
+        assertThat(role.cluster().check(DeleteExpiredDataAction.NAME, request), is(false));
+        assertThat(role.cluster().check(DeleteFilterAction.NAME, request), is(false));
+        assertThat(role.cluster().check(DeleteJobAction.NAME, request), is(false));
+        assertThat(role.cluster().check(DeleteModelSnapshotAction.NAME, request), is(false));
+        assertThat(role.cluster().check(FinalizeJobExecutionAction.NAME, request), is(false));
+        assertThat(role.cluster().check(FlushJobAction.NAME, request), is(false));
+        assertThat(role.cluster().check(GetBucketsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetCategoriesAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetDatafeedsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetDatafeedsStatsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetFiltersAction.NAME, request), is(false));
+        assertThat(role.cluster().check(GetInfluencersAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetJobsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetJobsStatsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetModelSnapshotsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetRecordsAction.NAME, request), is(true));
+        assertThat(role.cluster().check(IsolateDatafeedAction.NAME, request), is(false));
+        assertThat(role.cluster().check(KillProcessAction.NAME, request), is(false));
+        assertThat(role.cluster().check(OpenJobAction.NAME, request), is(false));
+        assertThat(role.cluster().check(PostDataAction.NAME, request), is(false));
+        assertThat(role.cluster().check(PreviewDatafeedAction.NAME, request), is(false));
+        assertThat(role.cluster().check(PutDatafeedAction.NAME, request), is(false));
+        assertThat(role.cluster().check(PutFilterAction.NAME, request), is(false));
+        assertThat(role.cluster().check(PutJobAction.NAME, request), is(false));
+        assertThat(role.cluster().check(RevertModelSnapshotAction.NAME, request), is(false));
+        assertThat(role.cluster().check(StartDatafeedAction.NAME, request), is(false));
+        assertThat(role.cluster().check(StopDatafeedAction.NAME, request), is(false));
+        assertThat(role.cluster().check(UpdateDatafeedAction.NAME, request), is(false));
+        assertThat(role.cluster().check(UpdateJobAction.NAME, request), is(false));
+        assertThat(role.cluster().check(UpdateModelSnapshotAction.NAME, request), is(false));
+        assertThat(role.cluster().check(UpdateProcessAction.NAME, request), is(false));
+        assertThat(role.cluster().check(ValidateDetectorAction.NAME, request), is(false));
+        assertThat(role.cluster().check(ValidateJobConfigAction.NAME, request), is(false));
         assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
         assertNoAccessAllowed(role, "foo");
@@ -649,19 +733,21 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testWatcherAdminRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("watcher_admin");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role role = Role.builder(roleDescriptor, null).build();
-        assertThat(role.cluster().check(PutWatchAction.NAME), is(true));
-        assertThat(role.cluster().check(GetWatchAction.NAME), is(true));
-        assertThat(role.cluster().check(DeleteWatchAction.NAME), is(true));
-        assertThat(role.cluster().check(ExecuteWatchAction.NAME), is(true));
-        assertThat(role.cluster().check(AckWatchAction.NAME), is(true));
-        assertThat(role.cluster().check(ActivateWatchAction.NAME), is(true));
-        assertThat(role.cluster().check(WatcherServiceAction.NAME), is(true));
-        assertThat(role.cluster().check(WatcherStatsAction.NAME), is(true));
+        assertThat(role.cluster().check(PutWatchAction.NAME, request), is(true));
+        assertThat(role.cluster().check(GetWatchAction.NAME, request), is(true));
+        assertThat(role.cluster().check(DeleteWatchAction.NAME, request), is(true));
+        assertThat(role.cluster().check(ExecuteWatchAction.NAME, request), is(true));
+        assertThat(role.cluster().check(AckWatchAction.NAME, request), is(true));
+        assertThat(role.cluster().check(ActivateWatchAction.NAME, request), is(true));
+        assertThat(role.cluster().check(WatcherServiceAction.NAME, request), is(true));
+        assertThat(role.cluster().check(WatcherStatsAction.NAME, request), is(true));
         assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
         assertThat(role.indices().allowedIndicesMatcher(IndexAction.NAME).test("foo"), is(false));
@@ -674,19 +760,21 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testWatcherUserRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("watcher_user");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role role = Role.builder(roleDescriptor, null).build();
-        assertThat(role.cluster().check(PutWatchAction.NAME), is(false));
-        assertThat(role.cluster().check(GetWatchAction.NAME), is(true));
-        assertThat(role.cluster().check(DeleteWatchAction.NAME), is(false));
-        assertThat(role.cluster().check(ExecuteWatchAction.NAME), is(false));
-        assertThat(role.cluster().check(AckWatchAction.NAME), is(false));
-        assertThat(role.cluster().check(ActivateWatchAction.NAME), is(false));
-        assertThat(role.cluster().check(WatcherServiceAction.NAME), is(false));
-        assertThat(role.cluster().check(WatcherStatsAction.NAME), is(true));
+        assertThat(role.cluster().check(PutWatchAction.NAME, request), is(false));
+        assertThat(role.cluster().check(GetWatchAction.NAME, request), is(true));
+        assertThat(role.cluster().check(DeleteWatchAction.NAME, request), is(false));
+        assertThat(role.cluster().check(ExecuteWatchAction.NAME, request), is(false));
+        assertThat(role.cluster().check(AckWatchAction.NAME, request), is(false));
+        assertThat(role.cluster().check(ActivateWatchAction.NAME, request), is(false));
+        assertThat(role.cluster().check(WatcherServiceAction.NAME, request), is(false));
+        assertThat(role.cluster().check(WatcherStatsAction.NAME, request), is(true));
         assertThat(role.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
         assertThat(role.indices().allowedIndicesMatcher(IndexAction.NAME).test("foo"), is(false));
@@ -724,15 +812,17 @@ public class ReservedRolesStoreTests extends ESTestCase {
     }
 
     public void testLogstashAdminRole() {
+        final TransportRequest request = mock(TransportRequest.class);
+
         RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor("logstash_admin");
         assertNotNull(roleDescriptor);
         assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true));
 
         Role logstashAdminRole = Role.builder(roleDescriptor, null).build();
-        assertThat(logstashAdminRole.cluster().check(ClusterHealthAction.NAME), is(false));
-        assertThat(logstashAdminRole.cluster().check(PutIndexTemplateAction.NAME), is(false));
-        assertThat(logstashAdminRole.cluster().check(ClusterRerouteAction.NAME), is(false));
-        assertThat(logstashAdminRole.cluster().check(ClusterUpdateSettingsAction.NAME), is(false));
+        assertThat(logstashAdminRole.cluster().check(ClusterHealthAction.NAME, request), is(false));
+        assertThat(logstashAdminRole.cluster().check(PutIndexTemplateAction.NAME, request), is(false));
+        assertThat(logstashAdminRole.cluster().check(ClusterRerouteAction.NAME, request), is(false));
+        assertThat(logstashAdminRole.cluster().check(ClusterUpdateSettingsAction.NAME, request), is(false));
 
         assertThat(logstashAdminRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(false));
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
index dbbaf7b0270..7a7deac0136 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
@@ -92,6 +92,7 @@ public class TransportPutDatafeedAction extends TransportMasterNodeAction<PutDat
                     .indices(request.getDatafeed().getIndices().toArray(new String[0]))
                     .privileges(SearchAction.NAME)
                     .build());
+            privRequest.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
 
             client.execute(HasPrivilegesAction.INSTANCE, privRequest, privResponseListener);
         } else {
@@ -107,8 +108,8 @@ public class TransportPutDatafeedAction extends TransportMasterNodeAction<PutDat
         } else {
             XContentBuilder builder = JsonXContent.contentBuilder();
             builder.startObject();
-            for (HasPrivilegesResponse.IndexPrivileges index : response.getIndexPrivileges()) {
-                builder.field(index.getIndex());
+            for (HasPrivilegesResponse.ResourcePrivileges index : response.getIndexPrivileges()) {
+                builder.field(index.getResource());
                 builder.map(index.getPrivileges());
             }
             builder.endObject();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index e483ad235c5..92784159700 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -84,6 +84,9 @@ import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.SecurityExtension;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.SecuritySettings;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction;
 import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction;
@@ -118,6 +121,9 @@ import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessCo
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapper;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
+import org.elasticsearch.xpack.security.action.privilege.TransportDeletePrivilegesAction;
+import org.elasticsearch.xpack.security.action.privilege.TransportGetPrivilegesAction;
+import org.elasticsearch.xpack.security.action.privilege.TransportPutPrivilegesAction;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
@@ -175,6 +181,7 @@ import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener;
 import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
@@ -182,6 +189,10 @@ import org.elasticsearch.xpack.security.rest.SecurityRestFilter;
 import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestInvalidateTokenAction;
+import org.elasticsearch.xpack.security.rest.action.privilege.RestDeletePrivilegesAction;
+import org.elasticsearch.xpack.security.rest.action.privilege.RestGetPrivilegesAction;
+import org.elasticsearch.xpack.security.rest.action.privilege.RestPutPrivilegeAction;
+import org.elasticsearch.xpack.security.rest.action.privilege.RestPutPrivilegesAction;
 import org.elasticsearch.xpack.security.rest.action.realm.RestClearRealmCacheAction;
 import org.elasticsearch.xpack.security.rest.action.role.RestClearRolesCacheAction;
 import org.elasticsearch.xpack.security.rest.action.role.RestDeleteRoleAction;
@@ -441,6 +452,9 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 anonymousUser, tokenService));
         components.add(authcService.get());
 
+        final NativePrivilegeStore privilegeStore = new NativePrivilegeStore(settings, client, securityIndex.get());
+        components.add(privilegeStore);
+
         final FileRolesStore fileRolesStore = new FileRolesStore(settings, env, resourceWatcherService, getLicenseState());
         final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client, getLicenseState(), securityIndex.get());
         final ReservedRolesStore reservedRolesStore = new ReservedRolesStore();
@@ -449,7 +463,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
             rolesProviders.addAll(extension.getRolesProviders(settings, resourceWatcherService));
         }
         final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore,
-            reservedRolesStore, rolesProviders, threadPool.getThreadContext(), getLicenseState());
+            reservedRolesStore, privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState());
         securityIndex.get().addIndexStateListener(allRolesStore::onSecurityIndexStateChange);
         // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be
         // minimal
@@ -697,7 +711,10 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 new ActionHandler<>(SamlPrepareAuthenticationAction.INSTANCE, TransportSamlPrepareAuthenticationAction.class),
                 new ActionHandler<>(SamlAuthenticateAction.INSTANCE, TransportSamlAuthenticateAction.class),
                 new ActionHandler<>(SamlLogoutAction.INSTANCE, TransportSamlLogoutAction.class),
-                new ActionHandler<>(SamlInvalidateSessionAction.INSTANCE, TransportSamlInvalidateSessionAction.class)
+                new ActionHandler<>(SamlInvalidateSessionAction.INSTANCE, TransportSamlInvalidateSessionAction.class),
+                new ActionHandler<>(GetPrivilegesAction.INSTANCE, TransportGetPrivilegesAction.class),
+                new ActionHandler<>(PutPrivilegesAction.INSTANCE, TransportPutPrivilegesAction.class),
+                new ActionHandler<>(DeletePrivilegesAction.INSTANCE, TransportDeletePrivilegesAction.class)
         );
     }
 
@@ -742,7 +759,11 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 new RestSamlPrepareAuthenticationAction(settings, restController, getLicenseState()),
                 new RestSamlAuthenticateAction(settings, restController, getLicenseState()),
                 new RestSamlLogoutAction(settings, restController, getLicenseState()),
-                new RestSamlInvalidateSessionAction(settings, restController, getLicenseState())
+                new RestSamlInvalidateSessionAction(settings, restController, getLicenseState()),
+                new RestGetPrivilegesAction(settings, restController, getLicenseState()),
+                new RestPutPrivilegesAction(settings, restController, getLicenseState()),
+                new RestPutPrivilegeAction(settings, restController, getLicenseState()),
+                new RestDeletePrivilegesAction(settings, restController, getLicenseState())
         );
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportDeletePrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportDeletePrivilegesAction.java
new file mode 100644
index 00000000000..da23ac50b69
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportDeletePrivilegesAction.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.privilege;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesResponse;
+import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
+
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * Transport action to retrieve one or more application privileges from the security index
+ */
+public class TransportDeletePrivilegesAction extends HandledTransportAction<DeletePrivilegesRequest, DeletePrivilegesResponse> {
+
+    private final NativePrivilegeStore privilegeStore;
+
+    @Inject
+    public TransportDeletePrivilegesAction(Settings settings, ActionFilters actionFilters,
+                                           NativePrivilegeStore privilegeStore,
+                                           TransportService transportService) {
+        super(settings, DeletePrivilegesAction.NAME, transportService, actionFilters, DeletePrivilegesRequest::new);
+        this.privilegeStore = privilegeStore;
+    }
+
+    @Override
+    protected void doExecute(Task task, final DeletePrivilegesRequest request, final ActionListener<DeletePrivilegesResponse> listener) {
+        if (request.privileges() == null || request.privileges().length == 0) {
+            listener.onResponse(new DeletePrivilegesResponse(Collections.emptyList()));
+            return;
+        }
+        final Set<String> names = Sets.newHashSet(request.privileges());
+        this.privilegeStore.deletePrivileges(request.application(), names, request.getRefreshPolicy(), ActionListener.wrap(
+                privileges -> listener.onResponse(
+                        new DeletePrivilegesResponse(privileges.getOrDefault(request.application(), Collections.emptyList()))
+                ), listener::onFailure
+        ));
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
new file mode 100644
index 00000000000..2a2f38fe7bb
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.privilege;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesResponse;
+import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.elasticsearch.common.Strings.isNullOrEmpty;
+
+/**
+ * Transport action to retrieve one or more application privileges from the security index
+ */
+public class TransportGetPrivilegesAction extends HandledTransportAction<GetPrivilegesRequest, GetPrivilegesResponse> {
+
+    private final NativePrivilegeStore privilegeStore;
+
+    @Inject
+    public TransportGetPrivilegesAction(Settings settings, ActionFilters actionFilters,
+                                        NativePrivilegeStore privilegeStore, TransportService transportService) {
+        super(settings, GetPrivilegesAction.NAME, transportService, actionFilters, GetPrivilegesRequest::new);
+        this.privilegeStore = privilegeStore;
+    }
+
+    @Override
+    protected void doExecute(Task task, final GetPrivilegesRequest request, final ActionListener<GetPrivilegesResponse> listener) {
+        final Set<String> names;
+        if (request.privileges() == null || request.privileges().length == 0) {
+            names = null;
+        } else {
+            names = new HashSet<>(Arrays.asList(request.privileges()));
+        }
+        final Collection<String> applications = isNullOrEmpty(request.application()) ? null : Collections.singleton(request.application());
+        this.privilegeStore.getPrivileges(applications, names, ActionListener.wrap(
+                privileges -> listener.onResponse(new GetPrivilegesResponse(privileges)),
+                listener::onFailure
+        ));
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java
new file mode 100644
index 00000000000..affb981c44f
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.privilege;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesResponse;
+import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
+
+import java.util.Collections;
+
+/**
+ * Transport action to retrieve one or more application privileges from the security index
+ */
+public class TransportPutPrivilegesAction extends HandledTransportAction<PutPrivilegesRequest, PutPrivilegesResponse> {
+
+    private final NativePrivilegeStore privilegeStore;
+
+    @Inject
+    public TransportPutPrivilegesAction(Settings settings, ActionFilters actionFilters,
+                                        NativePrivilegeStore privilegeStore, TransportService transportService) {
+        super(settings, PutPrivilegesAction.NAME, transportService, actionFilters, PutPrivilegesRequest::new);
+        this.privilegeStore = privilegeStore;
+    }
+
+    @Override
+    protected void doExecute(Task task, final PutPrivilegesRequest request, final ActionListener<PutPrivilegesResponse> listener) {
+        if (request.getPrivileges() == null || request.getPrivileges().size() == 0) {
+            listener.onResponse(new PutPrivilegesResponse(Collections.emptyMap()));
+        } else {
+            this.privilegeStore.putPrivileges(request.getPrivileges(), request.getRefreshPolicy(), ActionListener.wrap(
+                created -> listener.onResponse(new PutPrivilegesResponse(created)),
+                listener::onFailure
+            ));
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
index 9571b022e0a..b49984b28da 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
@@ -24,19 +24,26 @@ import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
+import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Transport action that tests whether a user has the specified
@@ -46,13 +53,16 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
 
     private final ThreadPool threadPool;
     private final AuthorizationService authorizationService;
+    private final NativePrivilegeStore privilegeStore;
 
     @Inject
     public TransportHasPrivilegesAction(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                        ActionFilters actionFilters, AuthorizationService authorizationService) {
+                                        ActionFilters actionFilters, AuthorizationService authorizationService,
+                                        NativePrivilegeStore privilegeStore) {
         super(settings, HasPrivilegesAction.NAME, transportService, actionFilters, HasPrivilegesRequest::new);
         this.threadPool = threadPool;
         this.authorizationService = authorizationService;
+        this.privilegeStore = privilegeStore;
     }
 
     @Override
@@ -66,15 +76,33 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
         }
 
         authorizationService.roles(user, ActionListener.wrap(
-                role -> checkPrivileges(request, role, listener),
-                listener::onFailure));
+            role -> resolveApplicationPrivileges(request, ActionListener.wrap(
+                applicationPrivilegeLookup -> checkPrivileges(request, role, applicationPrivilegeLookup, listener),
+                listener::onFailure)),
+            listener::onFailure));
+    }
+
+    private void resolveApplicationPrivileges(HasPrivilegesRequest request,
+                                              ActionListener<Collection<ApplicationPrivilegeDescriptor>> listener) {
+        final Set<String> applications = getApplicationNames(request);
+        privilegeStore.getPrivileges(applications, null, listener);
+    }
+
+    private Set<String> getApplicationNames(HasPrivilegesRequest request) {
+        return Arrays.stream(request.applicationPrivileges())
+            .map(RoleDescriptor.ApplicationResourcePrivileges::getApplication)
+            .collect(Collectors.toSet());
     }
 
     private void checkPrivileges(HasPrivilegesRequest request, Role userRole,
+                                 Collection<ApplicationPrivilegeDescriptor> applicationPrivileges,
                                  ActionListener<HasPrivilegesResponse> listener) {
-        logger.debug(() -> new ParameterizedMessage("Check whether role [{}] has privileges cluster=[{}] index=[{}]",
-                Strings.arrayToCommaDelimitedString(userRole.names()), Strings.arrayToCommaDelimitedString(request.clusterPrivileges()),
-                Strings.arrayToCommaDelimitedString(request.indexPrivileges())));
+        logger.trace(() -> new ParameterizedMessage("Check whether role [{}] has privileges cluster=[{}] index=[{}] application=[{}]",
+            Strings.arrayToCommaDelimitedString(userRole.names()),
+            Strings.arrayToCommaDelimitedString(request.clusterPrivileges()),
+            Strings.arrayToCommaDelimitedString(request.indexPrivileges()),
+            Strings.arrayToCommaDelimitedString(request.applicationPrivileges())
+        ));
 
         Map<String, Boolean> cluster = new HashMap<>();
         for (String checkAction : request.clusterPrivileges()) {
@@ -86,30 +114,62 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
 
         final Map<IndicesPermission.Group, Automaton> predicateCache = new HashMap<>();
 
-        final Map<String, HasPrivilegesResponse.IndexPrivileges> indices = new LinkedHashMap<>();
+        final Map<String, HasPrivilegesResponse.ResourcePrivileges> indices = new LinkedHashMap<>();
         for (RoleDescriptor.IndicesPrivileges check : request.indexPrivileges()) {
             for (String index : check.getIndices()) {
                 final Map<String, Boolean> privileges = new HashMap<>();
-                final HasPrivilegesResponse.IndexPrivileges existing = indices.get(index);
+                final HasPrivilegesResponse.ResourcePrivileges existing = indices.get(index);
                 if (existing != null) {
                     privileges.putAll(existing.getPrivileges());
                 }
                 for (String privilege : check.getPrivileges()) {
                     if (testIndexMatch(index, privilege, userRole, predicateCache)) {
-                        logger.debug(() -> new ParameterizedMessage("Role [{}] has [{}] on [{}]",
-                                Strings.arrayToCommaDelimitedString(userRole.names()), privilege, index));
+                        logger.debug(() -> new ParameterizedMessage("Role [{}] has [{}] on index [{}]",
+                            Strings.arrayToCommaDelimitedString(userRole.names()), privilege, index));
                         privileges.put(privilege, true);
                     } else {
-                        logger.debug(() -> new ParameterizedMessage("Role [{}] does not have [{}] on [{}]",
-                                Strings.arrayToCommaDelimitedString(userRole.names()), privilege, index));
+                        logger.debug(() -> new ParameterizedMessage("Role [{}] does not have [{}] on index [{}]",
+                            Strings.arrayToCommaDelimitedString(userRole.names()), privilege, index));
                         privileges.put(privilege, false);
                         allMatch = false;
                     }
                 }
-                indices.put(index, new HasPrivilegesResponse.IndexPrivileges(index, privileges));
+                indices.put(index, new HasPrivilegesResponse.ResourcePrivileges(index, privileges));
             }
         }
-        listener.onResponse(new HasPrivilegesResponse(allMatch, cluster, indices.values()));
+
+        final Map<String, Collection<HasPrivilegesResponse.ResourcePrivileges>> privilegesByApplication = new HashMap<>();
+        for (String applicationName : getApplicationNames(request)) {
+            logger.debug("Checking privileges for application {}", applicationName);
+            final Map<String, HasPrivilegesResponse.ResourcePrivileges> appPrivilegesByResource = new LinkedHashMap<>();
+            for (RoleDescriptor.ApplicationResourcePrivileges p : request.applicationPrivileges()) {
+                if (applicationName.equals(p.getApplication())) {
+                    for (String resource : p.getResources()) {
+                        final Map<String, Boolean> privileges = new HashMap<>();
+                        final HasPrivilegesResponse.ResourcePrivileges existing = appPrivilegesByResource.get(resource);
+                        if (existing != null) {
+                            privileges.putAll(existing.getPrivileges());
+                        }
+                        for (String privilege : p.getPrivileges()) {
+                            if (testResourceMatch(applicationName, resource, privilege, userRole, applicationPrivileges)) {
+                                logger.debug(() -> new ParameterizedMessage("Role [{}] has [{} {}] on resource [{}]",
+                                    Strings.arrayToCommaDelimitedString(userRole.names()), applicationName, privilege, resource));
+                                privileges.put(privilege, true);
+                            } else {
+                                logger.debug(() -> new ParameterizedMessage("Role [{}] does not have [{} {}] on resource [{}]",
+                                    Strings.arrayToCommaDelimitedString(userRole.names()), applicationName, privilege, resource));
+                                privileges.put(privilege, false);
+                                allMatch = false;
+                            }
+                        }
+                        appPrivilegesByResource.put(resource, new HasPrivilegesResponse.ResourcePrivileges(resource, privileges));
+                    }
+                }
+            }
+            privilegesByApplication.put(applicationName, appPrivilegesByResource.values());
+        }
+
+        listener.onResponse(new HasPrivilegesResponse(allMatch, cluster, indices.values(), privilegesByApplication));
     }
 
     private boolean testIndexMatch(String checkIndex, String checkPrivilegeName, Role userRole,
@@ -139,4 +199,17 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
     private static boolean testPrivilege(Privilege checkPrivilege, Automaton roleAutomaton) {
         return Operations.subsetOf(checkPrivilege.getAutomaton(), roleAutomaton);
     }
+
+    private boolean testResourceMatch(String application, String checkResource, String checkPrivilegeName, Role userRole,
+                                      Collection<ApplicationPrivilegeDescriptor> privileges) {
+        final Set<String> nameSet = Collections.singleton(checkPrivilegeName);
+        final ApplicationPrivilege checkPrivilege = ApplicationPrivilege.get(application, nameSet, privileges);
+        assert checkPrivilege.getApplication().equals(application)
+            : "Privilege " + checkPrivilege + " should have application " + application;
+        assert checkPrivilege.name().equals(nameSet)
+            : "Privilege " + checkPrivilege + " should have name " + nameSet;
+
+        return userRole.application().grants(checkPrivilege, checkResource);
+    }
+
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
index 09de32643ed..642bc167f7d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -178,8 +178,8 @@ public class AuthorizationService extends AbstractComponent {
 
         // first, we'll check if the action is a cluster action. If it is, we'll only check it against the cluster permissions
         if (ClusterPrivilege.ACTION_MATCHER.test(action)) {
-            ClusterPermission cluster = permission.cluster();
-            if (cluster.check(action) || checkSameUserPermissions(action, request, authentication)) {
+            final ClusterPermission cluster = permission.cluster();
+            if (cluster.check(action, request) || checkSameUserPermissions(action, request, authentication) ) {
                 putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_ALL);
                 auditTrail.accessGranted(authentication, action, request, permission.names());
                 return;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
index 1018ceeda09..beb2ca60fb2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
@@ -12,6 +12,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.cache.Cache;
 import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -28,7 +29,8 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCa
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
@@ -51,6 +53,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.elasticsearch.xpack.core.security.SecurityField.setting;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed;
@@ -80,6 +83,7 @@ public class CompositeRolesStore extends AbstractComponent {
     private final FileRolesStore fileRolesStore;
     private final NativeRolesStore nativeRolesStore;
     private final ReservedRolesStore reservedRolesStore;
+    private final NativePrivilegeStore privilegeStore;
     private final XPackLicenseState licenseState;
     private final Cache<Set<String>, Role> roleCache;
     private final Set<String> negativeLookupCache;
@@ -88,7 +92,7 @@ public class CompositeRolesStore extends AbstractComponent {
     private final List<BiConsumer<Set<String>, ActionListener<Set<RoleDescriptor>>>> customRolesProviders;
 
     public CompositeRolesStore(Settings settings, FileRolesStore fileRolesStore, NativeRolesStore nativeRolesStore,
-                               ReservedRolesStore reservedRolesStore,
+                               ReservedRolesStore reservedRolesStore, NativePrivilegeStore privilegeStore,
                                List<BiConsumer<Set<String>, ActionListener<Set<RoleDescriptor>>>> rolesProviders,
                                ThreadContext threadContext, XPackLicenseState licenseState) {
         super(settings);
@@ -98,6 +102,7 @@ public class CompositeRolesStore extends AbstractComponent {
         fileRolesStore.addListener(this::invalidateAll);
         this.nativeRolesStore = nativeRolesStore;
         this.reservedRolesStore = reservedRolesStore;
+        this.privilegeStore = privilegeStore;
         this.licenseState = licenseState;
         CacheBuilder<Set<String>, Role> builder = CacheBuilder.builder();
         final int cacheSize = CACHE_SIZE_SETTING.get(settings);
@@ -117,31 +122,33 @@ public class CompositeRolesStore extends AbstractComponent {
         } else {
             final long invalidationCounter = numInvalidation.get();
             roleDescriptors(roleNames, ActionListener.wrap(
-                    (descriptors) -> {
-                        final Role role;
+                    descriptors -> {
+                        final Set<RoleDescriptor> effectiveDescriptors;
                         if (licenseState.isDocumentAndFieldLevelSecurityAllowed()) {
-                            role = buildRoleFromDescriptors(descriptors, fieldPermissionsCache);
+                            effectiveDescriptors = descriptors;
                         } else {
-                            final Set<RoleDescriptor> filtered = descriptors.stream()
+                            effectiveDescriptors = descriptors.stream()
                                     .filter((rd) -> rd.isUsingDocumentOrFieldLevelSecurity() == false)
                                     .collect(Collectors.toSet());
-                            role = buildRoleFromDescriptors(filtered, fieldPermissionsCache);
                         }
-
-                        if (role != null) {
-                            try (ReleasableLock ignored = readLock.acquire()) {
-                                /* this is kinda spooky. We use a read/write lock to ensure we don't modify the cache if we hold the write
-                                 * lock (fetching stats for instance - which is kinda overkill?) but since we fetching stuff in an async
-                                 * fashion we need to make sure that if the cache got invalidated since we started the request we don't
-                                 * put a potential stale result in the cache, hence the numInvalidation.get() comparison to the number of
-                                 * invalidation when we started. we just try to be on the safe side and don't cache potentially stale
-                                 * results*/
-                                if (invalidationCounter == numInvalidation.get()) {
-                                    roleCache.computeIfAbsent(roleNames, (s) -> role);
+                        logger.trace("Building role from descriptors [{}] for names [{}]", effectiveDescriptors, roleNames);
+                        buildRoleFromDescriptors(effectiveDescriptors, fieldPermissionsCache, privilegeStore, ActionListener.wrap(role -> {
+                            if (role != null) {
+                                try (ReleasableLock ignored = readLock.acquire()) {
+                                    /* this is kinda spooky. We use a read/write lock to ensure we don't modify the cache if we hold
+                                     * the write lock (fetching stats for instance - which is kinda overkill?) but since we fetching
+                                     * stuff in an async fashion we need to make sure that if the cache got invalidated since we
+                                     * started the request we don't put a potential stale result in the cache, hence the
+                                     * numInvalidation.get() comparison to the number of invalidation when we started. we just try to
+                                     * be on the safe side and don't cache potentially stale results
+                                     */
+                                    if (invalidationCounter == numInvalidation.get()) {
+                                        roleCache.computeIfAbsent(roleNames, (s) -> role);
+                                    }
                                 }
                             }
-                        }
-                        roleActionListener.onResponse(role);
+                            roleActionListener.onResponse(role);
+                        }, roleActionListener::onFailure));
                     },
                     roleActionListener::onFailure));
         }
@@ -238,25 +245,36 @@ public class CompositeRolesStore extends AbstractComponent {
         return Sets.difference(roleNames, foundNames);
     }
 
-    public static Role buildRoleFromDescriptors(Set<RoleDescriptor> roleDescriptors, FieldPermissionsCache fieldPermissionsCache) {
+    public static void buildRoleFromDescriptors(Collection<RoleDescriptor> roleDescriptors, FieldPermissionsCache fieldPermissionsCache,
+                                                NativePrivilegeStore privilegeStore, ActionListener<Role> listener) {
         if (roleDescriptors.isEmpty()) {
-            return Role.EMPTY;
+            listener.onResponse(Role.EMPTY);
+            return;
         }
+
         Set<String> clusterPrivileges = new HashSet<>();
+        final List<ConditionalClusterPrivilege> conditionalClusterPrivileges = new ArrayList<>();
         Set<String> runAs = new HashSet<>();
         Map<Set<String>, MergeableIndicesPrivilege> indicesPrivilegesMap = new HashMap<>();
+
+        // Keyed by application + resource
+        Map<Tuple<String, Set<String>>, Set<String>> applicationPrivilegesMap = new HashMap<>();
+
         List<String> roleNames = new ArrayList<>(roleDescriptors.size());
         for (RoleDescriptor descriptor : roleDescriptors) {
             roleNames.add(descriptor.getName());
             if (descriptor.getClusterPrivileges() != null) {
                 clusterPrivileges.addAll(Arrays.asList(descriptor.getClusterPrivileges()));
             }
+            if (descriptor.getConditionalClusterPrivileges() != null) {
+                conditionalClusterPrivileges.addAll(Arrays.asList(descriptor.getConditionalClusterPrivileges()));
+            }
             if (descriptor.getRunAs() != null) {
                 runAs.addAll(Arrays.asList(descriptor.getRunAs()));
             }
             IndicesPrivileges[] indicesPrivileges = descriptor.getIndicesPrivileges();
             for (IndicesPrivileges indicesPrivilege : indicesPrivileges) {
-                Set<String> key = Sets.newHashSet(indicesPrivilege.getIndices());
+                Set<String> key = newHashSet(indicesPrivilege.getIndices());
                 // if a index privilege is an explicit denial, then we treat it as non-existent since we skipped these in the past when
                 // merging
                 final boolean isExplicitDenial =
@@ -274,19 +292,44 @@ public class CompositeRolesStore extends AbstractComponent {
                     });
                 }
             }
+            for (RoleDescriptor.ApplicationResourcePrivileges appPrivilege : descriptor.getApplicationPrivileges()) {
+                Tuple<String, Set<String>> key = new Tuple<>(appPrivilege.getApplication(), newHashSet(appPrivilege.getResources()));
+                applicationPrivilegesMap.compute(key, (k, v) -> {
+                    if (v == null) {
+                        return newHashSet(appPrivilege.getPrivileges());
+                    } else {
+                        v.addAll(Arrays.asList(appPrivilege.getPrivileges()));
+                        return v;
+                    }
+                });
+            }
         }
 
-        final Set<String> clusterPrivs = clusterPrivileges.isEmpty() ? null : clusterPrivileges;
         final Privilege runAsPrivilege = runAs.isEmpty() ? Privilege.NONE : new Privilege(runAs, runAs.toArray(Strings.EMPTY_ARRAY));
-        Role.Builder builder = Role.builder(roleNames.toArray(new String[roleNames.size()]))
-                .cluster(ClusterPrivilege.get(clusterPrivs))
+        final Role.Builder builder = Role.builder(roleNames.toArray(new String[roleNames.size()]))
+                .cluster(clusterPrivileges, conditionalClusterPrivileges)
                 .runAs(runAsPrivilege);
         indicesPrivilegesMap.entrySet().forEach((entry) -> {
             MergeableIndicesPrivilege privilege = entry.getValue();
             builder.add(fieldPermissionsCache.getFieldPermissions(privilege.fieldPermissionsDefinition), privilege.query,
                     IndexPrivilege.get(privilege.privileges), privilege.indices.toArray(Strings.EMPTY_ARRAY));
         });
-        return builder.build();
+
+        if (applicationPrivilegesMap.isEmpty()) {
+            listener.onResponse(builder.build());
+        } else {
+            final Set<String> applicationNames = applicationPrivilegesMap.keySet().stream()
+                    .map(Tuple::v1)
+                    .collect(Collectors.toSet());
+            final Set<String> applicationPrivilegeNames = applicationPrivilegesMap.values().stream()
+                    .flatMap(Collection::stream)
+                    .collect(Collectors.toSet());
+            privilegeStore.getPrivileges(applicationNames, applicationPrivilegeNames, ActionListener.wrap(appPrivileges -> {
+                applicationPrivilegesMap.forEach((key, names) ->
+                        builder.addApplicationPrivilege(ApplicationPrivilege.get(key.v1(), names, appPrivileges), key.v2()));
+                listener.onResponse(builder.build());
+            }, listener::onFailure));
+        }
     }
 
     public void invalidateAll() {
@@ -340,11 +383,11 @@ public class CompositeRolesStore extends AbstractComponent {
 
         MergeableIndicesPrivilege(String[] indices, String[] privileges, @Nullable String[] grantedFields, @Nullable String[] deniedFields,
                                   @Nullable BytesReference query) {
-            this.indices = Sets.newHashSet(Objects.requireNonNull(indices));
-            this.privileges = Sets.newHashSet(Objects.requireNonNull(privileges));
+            this.indices = newHashSet(Objects.requireNonNull(indices));
+            this.privileges = newHashSet(Objects.requireNonNull(privileges));
             this.fieldPermissionsDefinition = new FieldPermissionsDefinition(grantedFields, deniedFields);
             if (query != null) {
-                this.query = Sets.newHashSet(query);
+                this.query = newHashSet(query);
             }
         }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
new file mode 100644
index 00000000000..807cfff6c2c
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
@@ -0,0 +1,278 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authz.store;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
+import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParseException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.security.ScrollHelper;
+import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequest;
+import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheResponse;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.client.SecurityClient;
+import org.elasticsearch.xpack.security.support.SecurityIndexManager;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin;
+import static org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor.DOC_TYPE_VALUE;
+import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME;
+
+/**
+ * {@code NativePrivilegeStore} is a store that reads/writes {@link ApplicationPrivilegeDescriptor} objects,
+ * from an Elasticsearch index.
+ */
+public class NativePrivilegeStore extends AbstractComponent {
+
+    private static final Collector<Tuple<String, String>, ?, Map<String, List<String>>> TUPLES_TO_MAP = Collectors.toMap(
+        Tuple::v1,
+        t -> CollectionUtils.newSingletonArrayList(t.v2()), (a, b) -> {
+            a.addAll(b);
+            return a;
+        });
+
+    private final Client client;
+    private final SecurityClient securityClient;
+    private final SecurityIndexManager securityIndexManager;
+
+    public NativePrivilegeStore(Settings settings, Client client, SecurityIndexManager securityIndexManager) {
+        super(settings);
+        this.client = client;
+        this.securityClient = new SecurityClient(client);
+        this.securityIndexManager = securityIndexManager;
+    }
+
+    public void getPrivileges(Collection<String> applications, Collection<String> names,
+                              ActionListener<Collection<ApplicationPrivilegeDescriptor>> listener) {
+        if (applications != null && applications.size() == 1 && names != null && names.size() == 1) {
+            getPrivilege(Objects.requireNonNull(Iterables.get(applications, 0)), Objects.requireNonNull(Iterables.get(names, 0)),
+                ActionListener.wrap(privilege ->
+                        listener.onResponse(privilege == null ? Collections.emptyList() : Collections.singletonList(privilege)),
+                    listener::onFailure));
+        } else {
+            securityIndexManager.prepareIndexIfNeededThenExecute(listener::onFailure, () -> {
+                final QueryBuilder query;
+                final TermQueryBuilder typeQuery = QueryBuilders
+                    .termQuery(ApplicationPrivilegeDescriptor.Fields.TYPE.getPreferredName(), DOC_TYPE_VALUE);
+                if (isEmpty(applications) && isEmpty(names)) {
+                    query = typeQuery;
+                } else if (isEmpty(names)) {
+                    query = QueryBuilders.boolQuery().filter(typeQuery).filter(
+                        QueryBuilders.termsQuery(ApplicationPrivilegeDescriptor.Fields.APPLICATION.getPreferredName(), applications));
+                } else if (isEmpty(applications)) {
+                    query = QueryBuilders.boolQuery().filter(typeQuery)
+                        .filter(QueryBuilders.termsQuery(ApplicationPrivilegeDescriptor.Fields.NAME.getPreferredName(), names));
+                } else {
+                    final String[] docIds = applications.stream()
+                        .flatMap(a -> names.stream().map(n -> toDocId(a, n)))
+                        .toArray(String[]::new);
+                    query = QueryBuilders.boolQuery().filter(typeQuery).filter(QueryBuilders.idsQuery("doc").addIds(docIds));
+                }
+                final Supplier<ThreadContext.StoredContext> supplier = client.threadPool().getThreadContext().newRestorableContext(false);
+                try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) {
+                    SearchRequest request = client.prepareSearch(SECURITY_INDEX_NAME)
+                        .setScroll(TimeValue.timeValueSeconds(10L))
+                        .setQuery(query)
+                        .setSize(1000)
+                        .setFetchSource(true)
+                        .request();
+                    logger.trace(() ->
+                        new ParameterizedMessage("Searching for privileges [{}] with query [{}]", names, Strings.toString(query)));
+                    request.indicesOptions().ignoreUnavailable();
+                    ScrollHelper.fetchAllByEntity(client, request, new ContextPreservingActionListener<>(supplier, listener),
+                        hit -> buildPrivilege(hit.getId(), hit.getSourceRef()));
+                }
+            });
+        }
+    }
+
+    private static boolean isEmpty(Collection<String> collection) {
+        return collection == null || collection.isEmpty();
+    }
+
+    public void getPrivilege(String application, String name, ActionListener<ApplicationPrivilegeDescriptor> listener) {
+        securityIndexManager.prepareIndexIfNeededThenExecute(listener::onFailure,
+            () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN,
+                client.prepareGet(SECURITY_INDEX_NAME, "doc", toDocId(application, name)).request(),
+                new ActionListener<GetResponse>() {
+                    @Override
+                    public void onResponse(GetResponse response) {
+                        if (response.isExists()) {
+                            listener.onResponse(buildPrivilege(response.getId(), response.getSourceAsBytesRef()));
+                        } else {
+                            listener.onResponse(null);
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        // if the index or the shard is not there / available we just claim the privilege is not there
+                        if (TransportActions.isShardNotAvailableException(e)) {
+                            logger.warn(new ParameterizedMessage("failed to load privilege [{}] index not available", name), e);
+                            listener.onResponse(null);
+                        } else {
+                            logger.error(new ParameterizedMessage("failed to load privilege [{}]", name), e);
+                            listener.onFailure(e);
+                        }
+                    }
+                },
+                client::get));
+    }
+
+    public void putPrivileges(Collection<ApplicationPrivilegeDescriptor> privileges, WriteRequest.RefreshPolicy refreshPolicy,
+                              ActionListener<Map<String, List<String>>> listener) {
+        securityIndexManager.prepareIndexIfNeededThenExecute(listener::onFailure, () -> {
+            ActionListener<IndexResponse> groupListener = new GroupedActionListener<>(
+                ActionListener.wrap((Collection<IndexResponse> responses) -> {
+                    final Map<String, List<String>> createdNames = responses.stream()
+                        .filter(r -> r.getResult() == DocWriteResponse.Result.CREATED)
+                        .map(r -> r.getId())
+                        .map(NativePrivilegeStore::nameFromDocId)
+                        .collect(TUPLES_TO_MAP);
+                    clearRolesCache(listener, createdNames);
+                }, listener::onFailure), privileges.size(), Collections.emptyList());
+            for (ApplicationPrivilegeDescriptor privilege : privileges) {
+                innerPutPrivilege(privilege, refreshPolicy, groupListener);
+            }
+        });
+    }
+
+    private void innerPutPrivilege(ApplicationPrivilegeDescriptor privilege, WriteRequest.RefreshPolicy refreshPolicy,
+                                   ActionListener<IndexResponse> listener) {
+        try {
+            final String name = privilege.getName();
+            final XContentBuilder xContentBuilder = privilege.toXContent(jsonBuilder(), true);
+            ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN,
+                client.prepareIndex(SECURITY_INDEX_NAME, "doc", toDocId(privilege.getApplication(), name))
+                    .setSource(xContentBuilder)
+                    .setRefreshPolicy(refreshPolicy)
+                    .request(), listener, client::index);
+        } catch (Exception e) {
+            logger.warn("Failed to put privilege {} - {}", Strings.toString(privilege), e.toString());
+            listener.onFailure(e);
+        }
+
+    }
+
+    public void deletePrivileges(String application, Collection<String> names, WriteRequest.RefreshPolicy refreshPolicy,
+                                 ActionListener<Map<String, List<String>>> listener) {
+        securityIndexManager.prepareIndexIfNeededThenExecute(listener::onFailure, () -> {
+            ActionListener<DeleteResponse> groupListener = new GroupedActionListener<>(
+                ActionListener.wrap(responses -> {
+                    final Map<String, List<String>> deletedNames = responses.stream()
+                        .filter(r -> r.getResult() == DocWriteResponse.Result.DELETED)
+                        .map(r -> r.getId())
+                        .map(NativePrivilegeStore::nameFromDocId)
+                        .collect(TUPLES_TO_MAP);
+                    clearRolesCache(listener, deletedNames);
+                }, listener::onFailure), names.size(), Collections.emptyList());
+            for (String name : names) {
+                ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN,
+                    client.prepareDelete(SECURITY_INDEX_NAME, "doc", toDocId(application, name))
+                        .setRefreshPolicy(refreshPolicy)
+                        .request(), groupListener, client::delete);
+            }
+        });
+    }
+
+    private <T> void clearRolesCache(ActionListener<T> listener, T value) {
+        // This currently clears _all_ roles, but could be improved to clear only those roles that reference the affected application
+        ClearRolesCacheRequest request = new ClearRolesCacheRequest();
+        executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request,
+            new ActionListener<ClearRolesCacheResponse>() {
+                @Override
+                public void onResponse(ClearRolesCacheResponse nodes) {
+                    listener.onResponse(value);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    logger.error("unable to clear role cache", e);
+                    listener.onFailure(
+                        new ElasticsearchException("clearing the role cache failed. please clear the role cache manually", e));
+                }
+            }, securityClient::clearRolesCache);
+    }
+
+    private ApplicationPrivilegeDescriptor buildPrivilege(String docId, BytesReference source) {
+        logger.trace("Building privilege from [{}] [{}]", docId, source == null ? "<<null>>" : source.utf8ToString());
+        if (source == null) {
+            return null;
+        }
+        final Tuple<String, String> name = nameFromDocId(docId);
+        try {
+            // EMPTY is safe here because we never use namedObject
+
+            try (StreamInput input = source.streamInput();
+                 XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
+                     LoggingDeprecationHandler.INSTANCE, input)) {
+                final ApplicationPrivilegeDescriptor privilege = ApplicationPrivilegeDescriptor.parse(parser, null, null, true);
+                assert privilege.getApplication().equals(name.v1())
+                    : "Incorrect application name for privilege. Expected [" + name.v1() + "] but was " + privilege.getApplication();
+                assert privilege.getName().equals(name.v2())
+                    : "Incorrect name for application privilege. Expected [" + name.v2() + "] but was " + privilege.getName();
+                return privilege;
+            }
+        } catch (IOException | XContentParseException e) {
+            logger.error(new ParameterizedMessage("cannot parse application privilege [{}]", name), e);
+            return null;
+        }
+    }
+
+    private static Tuple<String, String> nameFromDocId(String docId) {
+        final String name = docId.substring(DOC_TYPE_VALUE.length() + 1);
+        assert name != null && name.length() > 0 : "Invalid name '" + name + "'";
+        final int colon = name.indexOf(':');
+        assert colon > 0 : "Invalid name '" + name + "' (missing colon)";
+        return new Tuple<>(name.substring(0, colon), name.substring(colon + 1));
+    }
+
+    private static String toDocId(String application, String name) {
+        return DOC_TYPE_VALUE + "_" + application + ":" + name;
+    }
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
index 9093b6a6673..e578a4005c4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
@@ -13,6 +13,7 @@ import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.MultiSearchResponse.Item;
@@ -63,6 +64,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin;
 import static org.elasticsearch.xpack.core.security.SecurityField.setting;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ROLE_TYPE;
+import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME;
 
 /**
  * NativeRolesStore is a {@code RolesStore} that, instead of reading from a
@@ -173,15 +175,17 @@ public class NativeRolesStore extends AbstractComponent {
                 listener.onFailure(e);
                 return;
             }
+            final IndexRequest indexRequest = client.prepareIndex(SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForUser(role.getName()))
+                    .setSource(xContentBuilder)
+                    .setRefreshPolicy(request.getRefreshPolicy())
+                    .request();
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN,
-                    client.prepareIndex(SecurityIndexManager.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForUser(role.getName()))
-                            .setSource(xContentBuilder)
-                            .setRefreshPolicy(request.getRefreshPolicy())
-                            .request(),
+                    indexRequest,
                     new ActionListener<IndexResponse>() {
                         @Override
                         public void onResponse(IndexResponse indexResponse) {
                             final boolean created = indexResponse.getResult() == DocWriteResponse.Result.CREATED;
+                            logger.trace("Created role: [{}]", indexRequest);
                             clearRoleCache(role.getName(), listener, created);
                         }
 
@@ -234,7 +238,6 @@ public class NativeRolesStore extends AbstractComponent {
                             } else {
                                 usageStats.put("size", responses[0].getResponse().getHits().getTotalHits());
                             }
-
                             if (responses[1].isFailure()) {
                                 usageStats.put("fls", false);
                             } else {
@@ -289,7 +292,7 @@ public class NativeRolesStore extends AbstractComponent {
     private void executeGetRoleRequest(String role, ActionListener<GetResponse> listener) {
         securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () ->
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN,
-                    client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME,
+                    client.prepareGet(SECURITY_INDEX_NAME,
                             ROLE_DOC_TYPE, getIdForUser(role)).request(),
                     listener,
                     client::get));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
index 0b2642ae5be..9006ec620b5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
@@ -56,7 +56,7 @@ public abstract class SecurityBaseRestHandler extends BaseRestHandler {
     /**
      * Check whether the given request is allowed within the current license state and setup,
      * and return the name of any unlicensed feature.
-     * By default this returns an exception is security is not available by the current license or
+     * By default this returns an exception if security is not available by the current license or
      * security is not enabled.
      * Sub-classes can override this method if they have additional requirements.
      *
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
new file mode 100644
index 00000000000..d0cee0dd6b9
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.rest.action.privilege;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesResponse;
+import org.elasticsearch.xpack.core.security.client.SecurityClient;
+import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+
+import static org.elasticsearch.rest.RestRequest.Method.DELETE;
+
+/**
+ * Rest action to delete one or more privileges from the security index
+ */
+public class RestDeletePrivilegesAction extends SecurityBaseRestHandler {
+
+    public RestDeletePrivilegesAction(Settings settings, RestController controller, XPackLicenseState licenseState) {
+        super(settings, licenseState);
+        controller.registerHandler(DELETE, "/_xpack/security/privilege/{application}/{privilege}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_security_delete_privilege_action";
+    }
+
+    @Override
+    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        final String application = request.param("application");
+        final String[] privileges = request.paramAsStringArray("privilege", null);
+        final String refresh = request.param("refresh");
+        return channel -> new SecurityClient(client).prepareDeletePrivileges(application, privileges)
+                .setRefreshPolicy(refresh)
+                .execute(new RestBuilderListener<DeletePrivilegesResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(DeletePrivilegesResponse response, XContentBuilder builder) throws Exception {
+                        builder.startObject();
+                        builder.startObject(application);
+                        for (String privilege : new HashSet<>(Arrays.asList(privileges))) {
+                            builder.field(privilege, Collections.singletonMap("found", response.found().contains(privilege)));
+                        }
+                        builder.endObject();
+                        builder.endObject();
+                        return new BytesRestResponse(response.found().isEmpty() ? RestStatus.NOT_FOUND : RestStatus.OK, builder);
+                    }
+                });
+    }
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
new file mode 100644
index 00000000000..8e3c3bbb87e
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.rest.action.privilege;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.client.SecurityClient;
+import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+/**
+ * Rest action to retrieve an application privilege from the security index
+ */
+public class RestGetPrivilegesAction extends SecurityBaseRestHandler {
+
+    public RestGetPrivilegesAction(Settings settings, RestController controller, XPackLicenseState licenseState) {
+        super(settings, licenseState);
+        controller.registerHandler(GET, "/_xpack/security/privilege/", this);
+        controller.registerHandler(GET, "/_xpack/security/privilege/{application}", this);
+        controller.registerHandler(GET, "/_xpack/security/privilege/{application}/{privilege}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_security_get_privileges_action";
+    }
+
+    @Override
+    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        final String application = request.param("application");
+        final String[] privileges = request.paramAsStringArray("privilege", Strings.EMPTY_ARRAY);
+
+        return channel -> new SecurityClient(client).prepareGetPrivileges(application, privileges)
+                .execute(new RestBuilderListener<GetPrivilegesResponse>(channel) {
+                    @Override
+                    public RestResponse buildResponse(GetPrivilegesResponse response, XContentBuilder builder) throws Exception {
+                        final Map<String, Set<ApplicationPrivilegeDescriptor>> privsByApp = groupByApplicationName(response.privileges());
+                        builder.startObject();
+                        for (String app : privsByApp.keySet()) {
+                            builder.startObject(app);
+                            for (ApplicationPrivilegeDescriptor privilege : privsByApp.get(app)) {
+                                builder.field(privilege.getName(), privilege);
+                            }
+                            builder.endObject();
+                        }
+                        builder.endObject();
+
+                        // if the user asked for specific privileges, but none of them were found
+                        // we'll return an empty result and 404 status code
+                        if (privileges.length != 0 && response.privileges().length == 0) {
+                            return new BytesRestResponse(RestStatus.NOT_FOUND, builder);
+                        }
+
+                        // either the user asked for all privileges, or at least one of the privileges
+                        // was found
+                        return new BytesRestResponse(RestStatus.OK, builder);
+                    }
+                });
+    }
+
+    static Map<String, Set<ApplicationPrivilegeDescriptor>> groupByApplicationName(ApplicationPrivilegeDescriptor[] privileges) {
+        return Arrays.stream(privileges).collect(Collectors.toMap(
+                ApplicationPrivilegeDescriptor::getApplication,
+                Collections::singleton,
+                Sets::union
+        ));
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegeAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegeAction.java
new file mode 100644
index 00000000000..6c3ef8e70fa
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegeAction.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.rest.action.privilege;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRequestBuilder;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.client.SecurityClient;
+import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
+
+import java.io.IOException;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
+/**
+ * Rest endpoint to add one or more {@link ApplicationPrivilege} objects to the security index
+ */
+public class RestPutPrivilegeAction extends SecurityBaseRestHandler {
+
+    public RestPutPrivilegeAction(Settings settings, RestController controller, XPackLicenseState licenseState) {
+        super(settings, licenseState);
+        controller.registerHandler(PUT, "/_xpack/security/privilege/{application}/{privilege}", this);
+        controller.registerHandler(POST, "/_xpack/security/privilege/{application}/{privilege}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_security_put_privilege_action";
+    }
+
+    @Override
+    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        final String application = request.param("application");
+        final String privilege = request.param("privilege");
+        PutPrivilegesRequestBuilder requestBuilder = new SecurityClient(client)
+                .preparePutPrivilege(application, privilege, request.requiredContent(), request.getXContentType())
+                .setRefreshPolicy(request.param("refresh"));
+
+        return RestPutPrivilegesAction.execute(requestBuilder);
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
new file mode 100644
index 00000000000..eb1104c9bc0
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.rest.action.privilege;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.client.SecurityClient;
+import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+/**
+ * Rest endpoint to add one or more {@link ApplicationPrivilege} objects to the security index
+ */
+public class RestPutPrivilegesAction extends SecurityBaseRestHandler {
+
+    public RestPutPrivilegesAction(Settings settings, RestController controller, XPackLicenseState licenseState) {
+        super(settings, licenseState);
+        controller.registerHandler(POST, "/_xpack/security/privilege/", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_security_put_privileges_action";
+    }
+
+    @Override
+    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        PutPrivilegesRequestBuilder requestBuilder = new SecurityClient(client)
+                .preparePutPrivileges(request.requiredContent(), request.getXContentType())
+                .setRefreshPolicy(request.param("refresh"));
+
+        return execute(requestBuilder);
+    }
+
+    static RestChannelConsumer execute(PutPrivilegesRequestBuilder requestBuilder) {
+        return channel -> requestBuilder.execute(new RestBuilderListener<PutPrivilegesResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(PutPrivilegesResponse response, XContentBuilder builder) throws Exception {
+                final List<ApplicationPrivilegeDescriptor> privileges = requestBuilder.request().getPrivileges();
+                Map<String, Map<String, Map<String, Boolean>>> result = new HashMap<>();
+                privileges.stream()
+                        .map(ApplicationPrivilegeDescriptor::getApplication)
+                        .distinct()
+                        .forEach(a -> result.put(a, new HashMap<>()));
+                privileges.forEach(privilege -> {
+                    String name = privilege.getName();
+                    boolean created = response.created().getOrDefault(privilege.getApplication(), Collections.emptyList()).contains(name);
+                    result.get(privilege.getApplication()).put(name, Collections.singletonMap("created", created));
+                });
+                builder.map(result);
+                return new BytesRestResponse(RestStatus.OK, builder);
+            }
+        });
+    }
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
index cc566c212cf..4949b18366d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
@@ -6,8 +6,11 @@
 package org.elasticsearch.xpack.security.rest.action.user;
 
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -24,6 +27,8 @@ import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -54,8 +59,8 @@ public class RestHasPrivilegesAction extends SecurityBaseRestHandler {
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
         final String username = getUsername(request);
-        HasPrivilegesRequestBuilder requestBuilder = new SecurityClient(client)
-                .prepareHasPrivileges(username, request.requiredContent(), request.getXContentType());
+        final Tuple<XContentType, BytesReference> content = request.contentOrSourceParam();
+        HasPrivilegesRequestBuilder requestBuilder = new SecurityClient(client).prepareHasPrivileges(username, content.v2(), content.v1());
         return channel -> requestBuilder.execute(new HasPrivilegesRestResponseBuilder(username, channel));
     }
 
@@ -84,10 +89,12 @@ public class RestHasPrivilegesAction extends SecurityBaseRestHandler {
             builder.field("cluster");
             builder.map(response.getClusterPrivileges());
 
-            builder.startObject("index");
-            for (HasPrivilegesResponse.IndexPrivileges index : response.getIndexPrivileges()) {
-                builder.field(index.getIndex());
-                builder.map(index.getPrivileges());
+            appendResources(builder, "index", response.getIndexPrivileges());
+
+            builder.startObject("application");
+            final Map<String, List<HasPrivilegesResponse.ResourcePrivileges>> appPrivileges = response.getApplicationPrivileges();
+            for (String app : appPrivileges.keySet()) {
+                appendResources(builder, app, appPrivileges.get(app));
             }
             builder.endObject();
 
@@ -95,5 +102,15 @@ public class RestHasPrivilegesAction extends SecurityBaseRestHandler {
             return new BytesRestResponse(RestStatus.OK, builder);
         }
 
+        private void appendResources(XContentBuilder builder, String field, List<HasPrivilegesResponse.ResourcePrivileges> privileges)
+                throws IOException {
+            builder.startObject(field);
+            for (HasPrivilegesResponse.ResourcePrivileges privilege : privileges) {
+                builder.field(privilege.getResource());
+                builder.map(privilege.getPrivileges());
+            }
+            builder.endObject();
+        }
+
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaSystemRoleIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaSystemRoleIntegTests.java
new file mode 100644
index 00000000000..65fa6027c62
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaSystemRoleIntegTests.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.integration;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+
+import java.util.Locale;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.hamcrest.Matchers.is;
+
+public class KibanaSystemRoleIntegTests extends SecurityIntegTestCase {
+
+    protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray());
+
+    @Override
+    public String configUsers() {
+        final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD));
+        return super.configUsers() +
+            "kibana_system:" + usersPasswdHashed;
+    }
+
+    @Override
+    public String configUsersRoles() {
+        return super.configUsersRoles() +
+                "kibana_system:kibana_system";
+    }
+
+
+    public void testCreateIndexDeleteInKibanaIndex() throws Exception {
+        final String index = randomBoolean()? ".kibana" : ".kibana-" + randomAlphaOfLengthBetween(1, 10).toLowerCase(Locale.ENGLISH);
+
+        if (randomBoolean()) {
+            CreateIndexResponse createIndexResponse = client().filterWithHeader(singletonMap("Authorization",
+                    UsernamePasswordToken.basicAuthHeaderValue("kibana_system", USERS_PASSWD)))
+                    .admin().indices().prepareCreate(index).get();
+            assertThat(createIndexResponse.isAcknowledged(), is(true));
+        }
+
+        IndexResponse response = client()
+                .filterWithHeader(singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_system", USERS_PASSWD)))
+                .prepareIndex()
+                .setIndex(index)
+                .setType("dashboard")
+                .setSource("foo", "bar")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
+
+        DeleteResponse deleteResponse = client()
+                .filterWithHeader(singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_system", USERS_PASSWD)))
+                .prepareDelete(index, "dashboard", response.getId())
+                .get();
+        assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
index cc080a846fa..b5b939e1744 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
@@ -20,7 +20,7 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.test.NativeRealmIntegTestCase;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 
 import java.util.Locale;
@@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class KibanaUserRoleIntegTests extends SecurityIntegTestCase {
+public class KibanaUserRoleIntegTests extends NativeRealmIntegTestCase {
 
     protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray());
 
@@ -154,25 +154,25 @@ public class KibanaUserRoleIntegTests extends SecurityIntegTestCase {
 
         if (randomBoolean()) {
             CreateIndexResponse createIndexResponse = client().filterWithHeader(singletonMap("Authorization",
-                    UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)))
-                    .admin().indices().prepareCreate(index).get();
+                UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)))
+                .admin().indices().prepareCreate(index).get();
             assertThat(createIndexResponse.isAcknowledged(), is(true));
         }
 
         IndexResponse response = client()
-                .filterWithHeader(singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)))
-                .prepareIndex()
-                .setIndex(index)
-                .setType("dashboard")
-                .setSource("foo", "bar")
-                .setRefreshPolicy(IMMEDIATE)
-                .get();
+            .filterWithHeader(singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)))
+            .prepareIndex()
+            .setIndex(index)
+            .setType("dashboard")
+            .setSource("foo", "bar")
+            .setRefreshPolicy(IMMEDIATE)
+            .get();
         assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
 
         DeleteResponse deleteResponse = client()
-                .filterWithHeader(singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)))
-                .prepareDelete(index, "dashboard", response.getId())
-                .get();
+            .filterWithHeader(singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)))
+            .prepareDelete(index, "dashboard", response.getId())
+            .get();
         assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java
new file mode 100644
index 00000000000..db0548c03ef
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class PutPrivilegesRequestBuilderTests extends ESTestCase {
+
+    public void testBuildRequestWithMultipleElements() throws Exception {
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        builder.source(new BytesArray("{ "
+                + "\"foo\":{"
+                + "  \"read\":{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] },"
+                + "  \"write\":{ \"application\":\"foo\", \"name\":\"write\", \"actions\":[ \"data:/write/*\", \"admin:*\" ] },"
+                + "  \"all\":{ \"application\":\"foo\", \"name\":\"all\", \"actions\":[ \"*\" ] }"
+                + " }, "
+                + "\"bar\":{"
+                + "  \"read\":{ \"application\":\"bar\", \"name\":\"read\", \"actions\":[ \"read/*\" ] },"
+                + "  \"write\":{ \"application\":\"bar\", \"name\":\"write\", \"actions\":[ \"write/*\" ] },"
+                + "  \"all\":{ \"application\":\"bar\", \"name\":\"all\", \"actions\":[ \"*\" ] }"
+                + " } "
+                + "}"), XContentType.JSON);
+        final List<ApplicationPrivilegeDescriptor> privileges = builder.request().getPrivileges();
+        assertThat(privileges, iterableWithSize(6));
+        assertThat(privileges, contains(
+                descriptor("foo", "read", "data:/read/*", "admin:/read/*"),
+                descriptor("foo", "write", "data:/write/*", "admin:*"),
+                descriptor("foo", "all", "*"),
+                descriptor("bar", "read", "read/*"),
+                descriptor("bar", "write", "write/*"),
+                descriptor("bar", "all", "*")
+        ));
+    }
+
+    private ApplicationPrivilegeDescriptor descriptor(String app, String name, String ... actions) {
+        return new ApplicationPrivilegeDescriptor(app, name, Sets.newHashSet(actions), Collections.emptyMap());
+    }
+
+    public void testBuildRequestFromJsonObject() throws Exception {
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        builder.source("foo", "read", new BytesArray(
+                "{  \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] }"
+        ), XContentType.JSON);
+        final List<ApplicationPrivilegeDescriptor> privileges = builder.request().getPrivileges();
+        assertThat(privileges, iterableWithSize(1));
+        assertThat(privileges, contains(descriptor("foo", "read", "data:/read/*", "admin:/read/*")));
+    }
+
+    public void testPrivilegeNameValidationOfSingleElement() throws Exception {
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+                builder.source("foo", "write", new BytesArray(
+                        "{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] }"
+                ), XContentType.JSON));
+        assertThat(exception.getMessage(), containsString("write"));
+        assertThat(exception.getMessage(), containsString("read"));
+    }
+
+    public void testApplicationNameValidationOfSingleElement() throws Exception {
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+                builder.source("bar", "read", new BytesArray(
+                        "{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] }"
+                ), XContentType.JSON));
+        assertThat(exception.getMessage(), containsString("foo"));
+        assertThat(exception.getMessage(), containsString("bar"));
+    }
+
+    public void testPrivilegeNameValidationOfMultipleElement() throws Exception {
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+                builder.source(new BytesArray("{ \"foo\":{"
+                        + "\"write\":{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[\"data:/read/*\",\"admin:/read/*\"] },"
+                        + "\"all\":{ \"application\":\"foo\", \"name\":\"all\", \"actions\":[ \"/*\" ] }"
+                        + "} }"), XContentType.JSON)
+        );
+        assertThat(exception.getMessage(), containsString("write"));
+        assertThat(exception.getMessage(), containsString("read"));
+    }
+
+    public void testApplicationNameValidationOfMultipleElement() throws Exception {
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+                builder.source(new BytesArray("{ \"bar\":{"
+                        + "\"read\":{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] },"
+                        + "\"write\":{ \"application\":\"foo\", \"name\":\"write\", \"actions\":[ \"data:/write/*\", \"admin:/*\" ] },"
+                        + "\"all\":{ \"application\":\"foo\", \"name\":\"all\", \"actions\":[ \"/*\" ] }"
+                        + "} }"), XContentType.JSON)
+        );
+        assertThat(exception.getMessage(), containsString("bar"));
+        assertThat(exception.getMessage(), containsString("foo"));
+    }
+
+    public void testInferApplicationNameAndPrivilegeName() throws Exception {
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        builder.source(new BytesArray("{ \"foo\":{"
+            + "\"read\":{ \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] },"
+            + "\"write\":{ \"actions\":[ \"data:/write/*\", \"admin:/*\" ] },"
+            + "\"all\":{ \"actions\":[ \"*\" ] }"
+            + "} }"), XContentType.JSON);
+        assertThat(builder.request().getPrivileges(), iterableWithSize(3));
+        for (ApplicationPrivilegeDescriptor p : builder.request().getPrivileges()) {
+            assertThat(p.getApplication(), equalTo("foo"));
+            assertThat(p.getName(), notNullValue());
+        }
+        assertThat(builder.request().getPrivileges().get(0).getName(), equalTo("read"));
+        assertThat(builder.request().getPrivileges().get(1).getName(), equalTo("write"));
+        assertThat(builder.request().getPrivileges().get(2).getName(), equalTo("all"));
+    }
+
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/HasPrivilegesRequestBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/HasPrivilegesRequestBuilderTests.java
index 2d53a3e6e86..0b9de2da332 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/HasPrivilegesRequestBuilderTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/HasPrivilegesRequestBuilderTests.java
@@ -114,6 +114,6 @@ public class HasPrivilegesRequestBuilderTests extends ESTestCase {
         final ElasticsearchParseException parseException = expectThrows(ElasticsearchParseException.class,
                 () -> builder.source("elastic", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON)
         );
-        assertThat(parseException.getMessage(), containsString("[index] and [cluster] are both missing"));
+        assertThat(parseException.getMessage(), containsString("[cluster,index,applications] are missing"));
     }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java
index 2c49c8d595e..a2e283e1b36 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java
@@ -15,43 +15,60 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.mock.orig.Mockito;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
-import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse.IndexPrivileges;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse.ResourcePrivileges;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
+import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
 
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.util.set.Sets.newHashSet;
+import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.iterableWithSize;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+@TestLogging("org.elasticsearch.xpack.security.action.user.TransportHasPrivilegesAction:TRACE," +
+        "org.elasticsearch.xpack.core.security.authz.permission.ApplicationPermission:DEBUG")
 public class TransportHasPrivilegesActionTests extends ESTestCase {
 
     private User user;
     private Role role;
     private TransportHasPrivilegesAction action;
+    private List<ApplicationPrivilegeDescriptor> applicationPrivileges;
 
     @Before
     public void setup() {
@@ -75,7 +92,19 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
             return null;
         }).when(authorizationService).roles(eq(user), any(ActionListener.class));
 
-        action = new TransportHasPrivilegesAction(settings, threadPool, transportService, mock(ActionFilters.class), authorizationService);
+        applicationPrivileges = new ArrayList<>();
+        NativePrivilegeStore privilegeStore = mock(NativePrivilegeStore.class);
+        Mockito.doAnswer(inv -> {
+            assertThat(inv.getArguments(), arrayWithSize(3));
+            ActionListener<List<ApplicationPrivilegeDescriptor>> listener
+                = (ActionListener<List<ApplicationPrivilegeDescriptor>>) inv.getArguments()[2];
+            logger.info("Privileges for ({}) are {}", Arrays.toString(inv.getArguments()), applicationPrivileges);
+            listener.onResponse(applicationPrivileges);
+            return null;
+        }).when(privilegeStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
+
+        action = new TransportHasPrivilegesAction(settings, threadPool, transportService, mock(ActionFilters.class), authorizationService,
+            privilegeStore);
     }
 
     /**
@@ -92,6 +121,7 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
                 .indices("academy")
                 .privileges(DeleteAction.NAME, IndexAction.NAME)
                 .build());
+        request.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
         final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture();
         action.doExecute(mock(Task.class), request, future);
 
@@ -103,8 +133,8 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
         assertThat(response.getClusterPrivileges().get(ClusterHealthAction.NAME), equalTo(true));
 
         assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(1));
-        final IndexPrivileges result = response.getIndexPrivileges().get(0);
-        assertThat(result.getIndex(), equalTo("academy"));
+        final ResourcePrivileges result = response.getIndexPrivileges().get(0);
+        assertThat(result.getResource(), equalTo("academy"));
         assertThat(result.getPrivileges().size(), equalTo(2));
         assertThat(result.getPrivileges().get(DeleteAction.NAME), equalTo(true));
         assertThat(result.getPrivileges().get(IndexAction.NAME), equalTo(true));
@@ -128,6 +158,7 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
                 .indices("academy", "initiative", "school")
                 .privileges("delete", "index", "manage")
                 .build());
+        request.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
         final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture();
         action.doExecute(mock(Task.class), request, future);
 
@@ -139,23 +170,23 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
         assertThat(response.getClusterPrivileges().get("manage"), equalTo(false));
         assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(3));
 
-        final IndexPrivileges academy = response.getIndexPrivileges().get(0);
-        final IndexPrivileges initiative = response.getIndexPrivileges().get(1);
-        final IndexPrivileges school = response.getIndexPrivileges().get(2);
+        final ResourcePrivileges academy = response.getIndexPrivileges().get(0);
+        final ResourcePrivileges initiative = response.getIndexPrivileges().get(1);
+        final ResourcePrivileges school = response.getIndexPrivileges().get(2);
 
-        assertThat(academy.getIndex(), equalTo("academy"));
+        assertThat(academy.getResource(), equalTo("academy"));
         assertThat(academy.getPrivileges().size(), equalTo(3));
         assertThat(academy.getPrivileges().get("index"), equalTo(true)); // explicit
         assertThat(academy.getPrivileges().get("delete"), equalTo(false));
         assertThat(academy.getPrivileges().get("manage"), equalTo(false));
 
-        assertThat(initiative.getIndex(), equalTo("initiative"));
+        assertThat(initiative.getResource(), equalTo("initiative"));
         assertThat(initiative.getPrivileges().size(), equalTo(3));
         assertThat(initiative.getPrivileges().get("index"), equalTo(true)); // implied by write
         assertThat(initiative.getPrivileges().get("delete"), equalTo(true)); // implied by write
         assertThat(initiative.getPrivileges().get("manage"), equalTo(false));
 
-        assertThat(school.getIndex(), equalTo("school"));
+        assertThat(school.getResource(), equalTo("school"));
         assertThat(school.getPrivileges().size(), equalTo(3));
         assertThat(school.getPrivileges().get("index"), equalTo(false));
         assertThat(school.getPrivileges().get("delete"), equalTo(false));
@@ -177,8 +208,8 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
                 .build(), Strings.EMPTY_ARRAY);
         assertThat(response.isCompleteMatch(), is(false));
         assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(1));
-        final IndexPrivileges result = response.getIndexPrivileges().get(0);
-        assertThat(result.getIndex(), equalTo("academy"));
+        final ResourcePrivileges result = response.getIndexPrivileges().get(0);
+        assertThat(result.getResource(), equalTo("academy"));
         assertThat(result.getPrivileges().size(), equalTo(2));
         assertThat(result.getPrivileges().get("read"), equalTo(false));
         assertThat(result.getPrivileges().get("write"), equalTo(false));
@@ -191,10 +222,20 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
      * <em>does the user have ___ privilege on a wildcard that covers (is a superset of) this pattern?</em>
      */
     public void testWildcardHandling() throws Exception {
+        final ApplicationPrivilege kibanaRead = defineApplicationPrivilege("kibana", "read",
+                "data:read/*", "action:login", "action:view/dashboard");
+        final ApplicationPrivilege kibanaWrite = defineApplicationPrivilege("kibana", "write",
+                "data:write/*", "action:login", "action:view/dashboard");
+        final ApplicationPrivilege kibanaAdmin = defineApplicationPrivilege("kibana", "admin",
+                "action:login", "action:manage/*");
+        final ApplicationPrivilege kibanaViewSpace = defineApplicationPrivilege("kibana", "view-space",
+                "action:login", "space:view/*");
         role = Role.builder("test3")
                 .add(IndexPrivilege.ALL, "logstash-*", "foo?")
                 .add(IndexPrivilege.READ, "abc*")
                 .add(IndexPrivilege.WRITE, "*xyz")
+                .addApplicationPrivilege(kibanaRead, Collections.singleton("*"))
+                .addApplicationPrivilege(kibanaViewSpace, newHashSet("space/engineering/*", "space/builds"))
                 .build();
 
         final HasPrivilegesRequest request = new HasPrivilegesRequest();
@@ -230,6 +271,20 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
                         .privileges("read", "write", "manage") // read = No, write = Yes (WRITE, "*xyz"), manage = No
                         .build()
         );
+
+        request.applicationPrivileges(
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                        .resources("*")
+                        .application("kibana")
+                        .privileges(Sets.union(kibanaRead.name(), kibanaWrite.name())) // read = Yes, write = No
+                        .build(),
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                        .resources("space/engineering/project-*", "space/*") // project-* = Yes, space/* = Not
+                        .application("kibana")
+                        .privileges("space:view/dashboard")
+                        .build()
+        );
+
         final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture();
         action.doExecute(mock(Task.class), request, future);
 
@@ -238,15 +293,28 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
         assertThat(response.isCompleteMatch(), is(false));
         assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(8));
         assertThat(response.getIndexPrivileges(), containsInAnyOrder(
-                new IndexPrivileges("logstash-2016-*", Collections.singletonMap("write", true)),
-                new IndexPrivileges("logstash-*", Collections.singletonMap("read", true)),
-                new IndexPrivileges("log*", Collections.singletonMap("manage", false)),
-                new IndexPrivileges("foo?", Collections.singletonMap("read", true)),
-                new IndexPrivileges("foo*", Collections.singletonMap("read", false)),
-                new IndexPrivileges("abcd*", mapBuilder().put("read", true).put("write", false).map()),
-                new IndexPrivileges("abc*xyz", mapBuilder().put("read", true).put("write", true).put("manage", false).map()),
-                new IndexPrivileges("a*xyz", mapBuilder().put("read", false).put("write", true).put("manage", false).map())
+                new ResourcePrivileges("logstash-2016-*", Collections.singletonMap("write", true)),
+                new ResourcePrivileges("logstash-*", Collections.singletonMap("read", true)),
+                new ResourcePrivileges("log*", Collections.singletonMap("manage", false)),
+                new ResourcePrivileges("foo?", Collections.singletonMap("read", true)),
+                new ResourcePrivileges("foo*", Collections.singletonMap("read", false)),
+                new ResourcePrivileges("abcd*", mapBuilder().put("read", true).put("write", false).map()),
+                new ResourcePrivileges("abc*xyz", mapBuilder().put("read", true).put("write", true).put("manage", false).map()),
+                new ResourcePrivileges("a*xyz", mapBuilder().put("read", false).put("write", true).put("manage", false).map())
         ));
+        assertThat(response.getApplicationPrivileges().entrySet(), Matchers.iterableWithSize(1));
+        final List<ResourcePrivileges> kibanaPrivileges = response.getApplicationPrivileges().get("kibana");
+        assertThat(kibanaPrivileges, Matchers.iterableWithSize(3));
+        assertThat(Strings.collectionToCommaDelimitedString(kibanaPrivileges), kibanaPrivileges, containsInAnyOrder(
+                new ResourcePrivileges("*", mapBuilder().put("read", true).put("write", false).map()),
+                new ResourcePrivileges("space/engineering/project-*", Collections.singletonMap("space:view/dashboard", true)),
+                new ResourcePrivileges("space/*", Collections.singletonMap("space:view/dashboard", false))
+        ));
+    }
+
+    private ApplicationPrivilege defineApplicationPrivilege(String app, String name, String ... actions) {
+        this.applicationPrivileges.add(new ApplicationPrivilegeDescriptor(app, name, newHashSet(actions), emptyMap()));
+        return new ApplicationPrivilege(app, name, actions);
     }
 
     public void testCheckingIndexPermissionsDefinedOnDifferentPatterns() throws Exception {
@@ -262,27 +330,152 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
         assertThat(response.isCompleteMatch(), is(false));
         assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(2));
         assertThat(response.getIndexPrivileges(), containsInAnyOrder(
-                new IndexPrivileges("apache-2016-12",
+                new ResourcePrivileges("apache-2016-12",
                         MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
                                 .put("index", true).put("delete", true).map()),
-                new IndexPrivileges("apache-2017-01",
+                new ResourcePrivileges("apache-2017-01",
                         MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
                                 .put("index", true).put("delete", false).map()
                 )
         ));
     }
 
+    public void testCheckingApplicationPrivilegesOnDifferentApplicationsAndResources() throws Exception {
+        final ApplicationPrivilege app1Read = defineApplicationPrivilege("app1", "read", "data:read/*");
+        final ApplicationPrivilege app1Write = defineApplicationPrivilege("app1", "write", "data:write/*");
+        final ApplicationPrivilege app1All = defineApplicationPrivilege("app1", "all", "*");
+        final ApplicationPrivilege app2Read = defineApplicationPrivilege("app2", "read", "data:read/*");
+        final ApplicationPrivilege app2Write = defineApplicationPrivilege("app2", "write", "data:write/*");
+        final ApplicationPrivilege app2All = defineApplicationPrivilege("app2", "all", "*");
+
+        role = Role.builder("test-role")
+                .addApplicationPrivilege(app1Read, Collections.singleton("foo/*"))
+                .addApplicationPrivilege(app1All, Collections.singleton("foo/bar/baz"))
+                .addApplicationPrivilege(app2Read, Collections.singleton("foo/bar/*"))
+                .addApplicationPrivilege(app2Write, Collections.singleton("*/bar/*"))
+                .build();
+
+        final HasPrivilegesResponse response = hasPrivileges(new RoleDescriptor.IndicesPrivileges[0],
+                new RoleDescriptor.ApplicationResourcePrivileges[]{
+                        RoleDescriptor.ApplicationResourcePrivileges.builder()
+                                .application("app1")
+                                .resources("foo/1", "foo/bar/2", "foo/bar/baz", "baz/bar/foo")
+                                .privileges("read", "write", "all")
+                                .build(),
+                        RoleDescriptor.ApplicationResourcePrivileges.builder()
+                                .application("app2")
+                                .resources("foo/1", "foo/bar/2", "foo/bar/baz", "baz/bar/foo")
+                                .privileges("read", "write", "all")
+                                .build()
+                }, Strings.EMPTY_ARRAY);
+
+        assertThat(response.isCompleteMatch(), is(false));
+        assertThat(response.getIndexPrivileges(), Matchers.emptyIterable());
+        assertThat(response.getApplicationPrivileges().entrySet(), Matchers.iterableWithSize(2));
+        final List<ResourcePrivileges> app1 = response.getApplicationPrivileges().get("app1");
+        assertThat(app1, Matchers.iterableWithSize(4));
+        assertThat(Strings.collectionToCommaDelimitedString(app1), app1, containsInAnyOrder(
+                new ResourcePrivileges("foo/1", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                        .put("read", true).put("write", false).put("all", false).map()),
+                new ResourcePrivileges("foo/bar/2", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                        .put("read", true).put("write", false).put("all", false).map()),
+                new ResourcePrivileges("foo/bar/baz", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                        .put("read", true).put("write", true).put("all", true).map()),
+                new ResourcePrivileges("baz/bar/foo", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                        .put("read", false).put("write", false).put("all", false).map())
+        ));
+        final List<ResourcePrivileges> app2 = response.getApplicationPrivileges().get("app2");
+        assertThat(app2, Matchers.iterableWithSize(4));
+        assertThat(Strings.collectionToCommaDelimitedString(app2), app2, containsInAnyOrder(
+                new ResourcePrivileges("foo/1", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                        .put("read", false).put("write", false).put("all", false).map()),
+                new ResourcePrivileges("foo/bar/2", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                        .put("read", true).put("write", true).put("all", false).map()),
+                new ResourcePrivileges("foo/bar/baz", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                        .put("read", true).put("write", true).put("all", false).map()),
+                new ResourcePrivileges("baz/bar/foo", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                        .put("read", false).put("write", true).put("all", false).map())
+        ));
+    }
+
+    public void testCheckingApplicationPrivilegesWithComplexNames() throws Exception {
+        final String appName = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(3, 10);
+        final String action1 = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(2, 5);
+        final String action2 = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(6, 9);
+
+        final ApplicationPrivilege priv1 = defineApplicationPrivilege(appName, action1, "DATA:read/*", "ACTION:" + action1);
+        final ApplicationPrivilege priv2 = defineApplicationPrivilege(appName, action2, "DATA:read/*", "ACTION:" + action2);
+
+        role = Role.builder("test-write")
+            .addApplicationPrivilege(priv1, Collections.singleton("user/*/name"))
+            .build();
+
+        final HasPrivilegesResponse response = hasPrivileges(
+            new RoleDescriptor.IndicesPrivileges[0],
+            new RoleDescriptor.ApplicationResourcePrivileges[]{
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                    .application(appName)
+                    .resources("user/hawkeye/name")
+                    .privileges("DATA:read/user/*", "ACTION:" + action1, "ACTION:" + action2, action1, action2)
+                    .build()
+            },
+            "monitor");
+        assertThat(response.isCompleteMatch(), is(false));
+        assertThat(response.getApplicationPrivileges().keySet(), containsInAnyOrder(appName));
+        assertThat(response.getApplicationPrivileges().get(appName), iterableWithSize(1));
+        assertThat(response.getApplicationPrivileges().get(appName), containsInAnyOrder(
+            new ResourcePrivileges("user/hawkeye/name", MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("DATA:read/user/*", true)
+                .put("ACTION:" + action1, true)
+                .put("ACTION:" + action2, false)
+                .put(action1, true)
+                .put(action2, false)
+                .map())
+        ));
+    }
+
     public void testIsCompleteMatch() throws Exception {
+        final ApplicationPrivilege kibanaRead = defineApplicationPrivilege("kibana", "read", "data:read/*");
+        final ApplicationPrivilege kibanaWrite = defineApplicationPrivilege("kibana", "write", "data:write/*");
         role = Role.builder("test-write")
                 .cluster(ClusterPrivilege.MONITOR)
                 .add(IndexPrivilege.READ, "read-*")
                 .add(IndexPrivilege.ALL, "all-*")
+                .addApplicationPrivilege(kibanaRead, Collections.singleton("*"))
                 .build();
 
         assertThat(hasPrivileges(indexPrivileges("read", "read-123", "read-456", "all-999"), "monitor").isCompleteMatch(), is(true));
         assertThat(hasPrivileges(indexPrivileges("read", "read-123", "read-456", "all-999"), "manage").isCompleteMatch(), is(false));
         assertThat(hasPrivileges(indexPrivileges("write", "read-123", "read-456", "all-999"), "monitor").isCompleteMatch(), is(false));
         assertThat(hasPrivileges(indexPrivileges("write", "read-123", "read-456", "all-999"), "manage").isCompleteMatch(), is(false));
+        assertThat(hasPrivileges(
+                new RoleDescriptor.IndicesPrivileges[]{
+                        RoleDescriptor.IndicesPrivileges.builder()
+                                .indices("read-a")
+                                .privileges("read")
+                                .build(),
+                        RoleDescriptor.IndicesPrivileges.builder()
+                                .indices("all-b")
+                                .privileges("read", "write")
+                                .build()
+                },
+                new RoleDescriptor.ApplicationResourcePrivileges[]{
+                        RoleDescriptor.ApplicationResourcePrivileges.builder()
+                                .application("kibana")
+                                .resources("*")
+                                .privileges("read")
+                                .build()
+                },
+                "monitor").isCompleteMatch(), is(true));
+        assertThat(hasPrivileges(
+                new RoleDescriptor.IndicesPrivileges[]{indexPrivileges("read", "read-123", "read-456", "all-999")},
+                new RoleDescriptor.ApplicationResourcePrivileges[]{
+                        RoleDescriptor.ApplicationResourcePrivileges.builder()
+                                .application("kibana").resources("*").privileges("read").build(),
+                        RoleDescriptor.ApplicationResourcePrivileges.builder()
+                                .application("kibana").resources("*").privileges("write").build()
+                },
+                "monitor").isCompleteMatch(), is(false));
     }
 
     private RoleDescriptor.IndicesPrivileges indexPrivileges(String priv, String... indices) {
@@ -294,10 +487,21 @@ public class TransportHasPrivilegesActionTests extends ESTestCase {
 
     private HasPrivilegesResponse hasPrivileges(RoleDescriptor.IndicesPrivileges indicesPrivileges, String... clusterPrivileges)
             throws Exception {
+        return hasPrivileges(
+                new RoleDescriptor.IndicesPrivileges[]{indicesPrivileges},
+                new RoleDescriptor.ApplicationResourcePrivileges[0],
+                clusterPrivileges
+        );
+    }
+
+    private HasPrivilegesResponse hasPrivileges(RoleDescriptor.IndicesPrivileges[] indicesPrivileges,
+                                                RoleDescriptor.ApplicationResourcePrivileges[] appPrivileges,
+                                                String... clusterPrivileges) throws Exception {
         final HasPrivilegesRequest request = new HasPrivilegesRequest();
         request.username(user.principal());
         request.clusterPrivileges(clusterPrivileges);
         request.indexPrivileges(indicesPrivileges);
+        request.applicationPrivileges(appPrivileges);
         final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture();
         action.doExecute(mock(Task.class), request, future);
         final HasPrivilegesResponse response = future.get();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
index 7d446913368..453820ea519 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
@@ -79,7 +79,7 @@ public class AuditTrailTests extends SecurityIntegTestCase {
     public String configUsersRoles() {
         return super.configUsersRoles()
                 + ROLE_CAN_RUN_AS + ":" + AUTHENTICATE_USER + "\n"
-                + "kibana_user:" + EXECUTE_USER;
+                + "monitoring_user:" + EXECUTE_USER;
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java
index c42353ee752..e94cdff4232 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java
@@ -47,7 +47,7 @@ public class ESNativeRealmMigrateToolTests extends CommandTestCase {
                     @Override
                     protected Environment createEnv(Map<String, String> settings) throws UserException {
                         Settings.Builder builder = Settings.builder();
-                        settings.forEach((k,v) -> builder.put(k, v));
+                        settings.forEach((k, v) -> builder.put(k, v));
                         return TestEnvironment.newEnvironment(builder.build());
                     }
 
@@ -75,9 +75,11 @@ public class ESNativeRealmMigrateToolTests extends CommandTestCase {
         String[] runAs = Strings.EMPTY_ARRAY;
         RoleDescriptor rd = new RoleDescriptor("rolename", cluster, ips, runAs);
         assertThat(ESNativeRealmMigrateTool.MigrateUserOrRoles.createRoleJson(rd),
-                equalTo("{\"cluster\":[],\"indices\":[{\"names\":[\"i1\",\"i2\",\"i3\"]," +
-                                "\"privileges\":[\"all\"],\"field_security\":{\"grant\":[\"body\"]}}]," +
-                                "\"run_as\":[],\"metadata\":{},\"type\":\"role\"}"));
+                equalTo("{\"cluster\":[]," +
+                        "\"indices\":[{\"names\":[\"i1\",\"i2\",\"i3\"]," +
+                        "\"privileges\":[\"all\"],\"field_security\":{\"grant\":[\"body\"]}}]," +
+                        "\"applications\":[]," +
+                        "\"run_as\":[],\"metadata\":{},\"type\":\"role\"}"));
     }
 
     public void testTerminalLogger() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
index 2b5f81cac01..b7cd23745b9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
@@ -22,6 +22,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
@@ -66,6 +67,7 @@ import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
 
 /**
  * Tests for the NativeUsersStore and NativeRolesStore
@@ -358,10 +360,11 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
                 assertThat(e.status(), is(RestStatus.FORBIDDEN));
             }
         } else {
+            final TransportRequest request = mock(TransportRequest.class);
             GetRolesResponse getRolesResponse = c.prepareGetRoles().names("test_role").get();
             assertTrue("test_role does not exist!", getRolesResponse.hasRoles());
             assertTrue("any cluster permission should be authorized",
-                    Role.builder(getRolesResponse.roles()[0], null).build().cluster().check("cluster:admin/foo"));
+                    Role.builder(getRolesResponse.roles()[0], null).build().cluster().check("cluster:admin/foo", request));
 
             c.preparePutRole("test_role")
                     .cluster("none")
@@ -372,7 +375,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
             assertTrue("test_role does not exist!", getRolesResponse.hasRoles());
 
             assertFalse("no cluster permission should be authorized",
-                    Role.builder(getRolesResponse.roles()[0], null).build().cluster().check("cluster:admin/bar"));
+                    Role.builder(getRolesResponse.roles()[0], null).build().cluster().check("cluster:admin/bar", request));
         }
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
index 9e33e145e24..7722a9d2166 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
@@ -90,6 +90,8 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportActionProxy;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequestBuilder;
@@ -113,6 +115,9 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivile
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
@@ -122,6 +127,7 @@ import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.elasticsearch.xpack.sql.action.SqlQueryAction;
 import org.elasticsearch.xpack.sql.action.SqlQueryRequest;
 import org.junit.Before;
@@ -129,12 +135,15 @@ import org.mockito.Mockito;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.function.Predicate;
 
 import static java.util.Arrays.asList;
 import static org.elasticsearch.test.SecurityTestsUtils.assertAuthenticationException;
@@ -174,8 +183,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         rolesStore = mock(CompositeRolesStore.class);
         clusterService = mock(ClusterService.class);
         final Settings settings = Settings.builder()
-                .put("search.remote.other_cluster.seeds", "localhost:9999")
-                .build();
+            .put("search.remote.other_cluster.seeds", "localhost:9999")
+            .build();
         final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         auditTrail = mock(AuditTrailService.class);
@@ -183,9 +192,20 @@ public class AuthorizationServiceTests extends ESTestCase {
         threadPool = mock(ThreadPool.class);
         when(threadPool.getThreadContext()).thenReturn(threadContext);
         final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(settings);
+
+        final NativePrivilegeStore privilegesStore = mock(NativePrivilegeStore.class);
+        doAnswer(i -> {
+                assertThat(i.getArguments().length, equalTo(3));
+                final Object arg2 = i.getArguments()[2];
+                assertThat(arg2, instanceOf(ActionListener.class));
+                ActionListener<Collection<ApplicationPrivilege>> listener = (ActionListener<Collection<ApplicationPrivilege>>) arg2;
+                listener.onResponse(Collections.emptyList());
+                return null;
+            }
+        ).when(privilegesStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
+
         doAnswer((i) -> {
-            ActionListener<Role> callback =
-                    (ActionListener<Role>) i.getArguments()[2];
+            ActionListener<Role> callback = (ActionListener<Role>) i.getArguments()[2];
             Set<String> names = (Set<String>) i.getArguments()[0];
             assertNotNull(names);
             Set<RoleDescriptor> roleDescriptors = new HashSet<>();
@@ -199,22 +219,23 @@ public class AuthorizationServiceTests extends ESTestCase {
             if (roleDescriptors.isEmpty()) {
                 callback.onResponse(Role.EMPTY);
             } else {
-                callback.onResponse(
-                        CompositeRolesStore.buildRoleFromDescriptors(roleDescriptors, fieldPermissionsCache));
+                CompositeRolesStore.buildRoleFromDescriptors(roleDescriptors, fieldPermissionsCache, privilegesStore,
+                    ActionListener.wrap(r -> callback.onResponse(r), callback::onFailure)
+                );
             }
             return Void.TYPE;
         }).when(rolesStore).roles(any(Set.class), any(FieldPermissionsCache.class), any(ActionListener.class));
         authorizationService = new AuthorizationService(settings, rolesStore, clusterService,
-                auditTrail, new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(settings));
+            auditTrail, new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(settings));
     }
 
     private void authorize(Authentication authentication, String action, TransportRequest request) {
         PlainActionFuture<Void> future = new PlainActionFuture<>();
         AuthorizationUtils.AsyncAuthorizer authorizer = new AuthorizationUtils.AsyncAuthorizer(authentication, future,
-                (userRoles, runAsRoles) -> {
-                    authorizationService.authorize(authentication, action, request, userRoles, runAsRoles);
-                    future.onResponse(null);
-                });
+            (userRoles, runAsRoles) -> {
+                authorizationService.authorize(authentication, action, request, userRoles, runAsRoles);
+                future.onResponse(null);
+            });
         authorizer.authorize(authorizationService);
         future.actionGet();
     }
@@ -226,11 +247,11 @@ public class AuthorizationServiceTests extends ESTestCase {
         Authentication authentication = createAuthentication(SystemUser.INSTANCE);
         authorize(authentication, "indices:monitor/whatever", request);
         verify(auditTrail).accessGranted(authentication, "indices:monitor/whatever", request,
-            new String[] { SystemUser.ROLE_NAME });
+            new String[]{SystemUser.ROLE_NAME});
 
         authentication = createAuthentication(SystemUser.INSTANCE);
         authorize(authentication, "internal:whatever", request);
-        verify(auditTrail).accessGranted(authentication, "internal:whatever", request, new String[] { SystemUser.ROLE_NAME });
+        verify(auditTrail).accessGranted(authentication, "internal:whatever", request, new String[]{SystemUser.ROLE_NAME});
         verifyNoMoreInteractions(auditTrail);
     }
 
@@ -238,9 +259,9 @@ public class AuthorizationServiceTests extends ESTestCase {
         final TransportRequest request = mock(TransportRequest.class);
         final Authentication authentication = createAuthentication(SystemUser.INSTANCE);
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, "indices:", request),
-                "indices:", SystemUser.INSTANCE.principal());
-        verify(auditTrail).accessDenied(authentication, "indices:", request, new String[] { SystemUser.ROLE_NAME });
+            () -> authorize(authentication, "indices:", request),
+            "indices:", SystemUser.INSTANCE.principal());
+        verify(auditTrail).accessDenied(authentication, "indices:", request, new String[]{SystemUser.ROLE_NAME});
         verifyNoMoreInteractions(auditTrail);
     }
 
@@ -248,10 +269,10 @@ public class AuthorizationServiceTests extends ESTestCase {
         final TransportRequest request = mock(TransportRequest.class);
         final Authentication authentication = createAuthentication(SystemUser.INSTANCE);
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, "cluster:admin/whatever", request),
-                "cluster:admin/whatever", SystemUser.INSTANCE.principal());
+            () -> authorize(authentication, "cluster:admin/whatever", request),
+            "cluster:admin/whatever", SystemUser.INSTANCE.principal());
         verify(auditTrail).accessDenied(authentication, "cluster:admin/whatever", request,
-            new String[] { SystemUser.ROLE_NAME });
+            new String[]{SystemUser.ROLE_NAME});
         verifyNoMoreInteractions(auditTrail);
     }
 
@@ -259,10 +280,50 @@ public class AuthorizationServiceTests extends ESTestCase {
         final TransportRequest request = mock(TransportRequest.class);
         final Authentication authentication = createAuthentication(SystemUser.INSTANCE);
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, "cluster:admin/snapshot/status", request),
-                "cluster:admin/snapshot/status", SystemUser.INSTANCE.principal());
+            () -> authorize(authentication, "cluster:admin/snapshot/status", request),
+            "cluster:admin/snapshot/status", SystemUser.INSTANCE.principal());
         verify(auditTrail).accessDenied(authentication, "cluster:admin/snapshot/status", request,
-                new String[] { SystemUser.ROLE_NAME });
+            new String[]{SystemUser.ROLE_NAME});
+        verifyNoMoreInteractions(auditTrail);
+    }
+
+    public void testAuthorizeUsingConditionalPrivileges() {
+        final DeletePrivilegesRequest request = new DeletePrivilegesRequest();
+        final Authentication authentication = createAuthentication(new User("user1", "role1"));
+
+        final ConditionalClusterPrivilege conditionalClusterPrivilege = Mockito.mock(ConditionalClusterPrivilege.class);
+        final Predicate<TransportRequest> requestPredicate = r -> r == request;
+        Mockito.when(conditionalClusterPrivilege.getRequestPredicate()).thenReturn(requestPredicate);
+        Mockito.when(conditionalClusterPrivilege.getPrivilege()).thenReturn(ClusterPrivilege.MANAGE_SECURITY);
+        final ConditionalClusterPrivilege[] conditionalClusterPrivileges = new ConditionalClusterPrivilege[] {
+            conditionalClusterPrivilege
+        };
+        RoleDescriptor role = new RoleDescriptor("role1", null, null, null, conditionalClusterPrivileges, null, null ,null);
+        roleMap.put("role1", role);
+
+        authorize(authentication, DeletePrivilegesAction.NAME, request);
+        verify(auditTrail).accessGranted(authentication, DeletePrivilegesAction.NAME, request, new String[]{role.getName()});
+        verifyNoMoreInteractions(auditTrail);
+    }
+
+    public void testAuthorizationDeniedWhenConditionalPrivilegesDoNotMatch() {
+        final DeletePrivilegesRequest request = new DeletePrivilegesRequest();
+        final Authentication authentication = createAuthentication(new User("user1", "role1"));
+
+        final ConditionalClusterPrivilege conditionalClusterPrivilege = Mockito.mock(ConditionalClusterPrivilege.class);
+        final Predicate<TransportRequest> requestPredicate = r -> false;
+        Mockito.when(conditionalClusterPrivilege.getRequestPredicate()).thenReturn(requestPredicate);
+        Mockito.when(conditionalClusterPrivilege.getPrivilege()).thenReturn(ClusterPrivilege.MANAGE_SECURITY);
+        final ConditionalClusterPrivilege[] conditionalClusterPrivileges = new ConditionalClusterPrivilege[] {
+            conditionalClusterPrivilege
+        };
+        RoleDescriptor role = new RoleDescriptor("role1", null, null, null, conditionalClusterPrivileges, null, null ,null);
+        roleMap.put("role1", role);
+
+        assertThrowsAuthorizationException(
+            () -> authorize(authentication, DeletePrivilegesAction.NAME, request),
+            DeletePrivilegesAction.NAME, "user1");
+        verify(auditTrail).accessDenied(authentication, DeletePrivilegesAction.NAME, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
@@ -271,8 +332,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         final Authentication authentication = createAuthentication(new User("test user"));
         mockEmptyMetaData();
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, "indices:a", request),
-                "indices:a", "test user");
+            () -> authorize(authentication, "indices:a", request),
+            "indices:a", "test user");
         verify(auditTrail).accessDenied(authentication, "indices:a", request, Role.EMPTY.names());
         verifyNoMoreInteractions(auditTrail);
     }
@@ -298,8 +359,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         final Authentication authentication = createAuthentication(new User("test user"));
         mockEmptyMetaData();
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, SearchAction.NAME, request),
-                SearchAction.NAME, "test user");
+            () -> authorize(authentication, SearchAction.NAME, request),
+            SearchAction.NAME, "test user");
         verify(auditTrail).accessDenied(authentication, SearchAction.NAME, request, Role.EMPTY.names());
         verifyNoMoreInteractions(auditTrail);
     }
@@ -314,8 +375,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         final Authentication authentication = createAuthentication(new User("test user"));
         mockEmptyMetaData();
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, SearchAction.NAME, request),
-                SearchAction.NAME, "test user");
+            () -> authorize(authentication, SearchAction.NAME, request),
+            SearchAction.NAME, "test user");
         verify(auditTrail).accessDenied(authentication, SearchAction.NAME, request, Role.EMPTY.names());
         verifyNoMoreInteractions(auditTrail);
     }
@@ -325,12 +386,11 @@ public class AuthorizationServiceTests extends ESTestCase {
         Authentication authentication = createAuthentication(new User("test user"));
         mockEmptyMetaData();
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, SqlQueryAction.NAME, request),
-                SqlQueryAction.NAME, "test user");
+            () -> authorize(authentication, SqlQueryAction.NAME, request),
+            SqlQueryAction.NAME, "test user");
         verify(auditTrail).accessDenied(authentication, SqlQueryAction.NAME, request, Role.EMPTY.names());
         verifyNoMoreInteractions(auditTrail);
     }
-
     /**
      * Verifies that the behaviour tested in {@link #testUserWithNoRolesCanPerformRemoteSearch}
      * does not work for requests that are not remote-index-capable.
@@ -341,24 +401,24 @@ public class AuthorizationServiceTests extends ESTestCase {
         final Authentication authentication = createAuthentication(new User("test user"));
         mockEmptyMetaData();
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, DeleteIndexAction.NAME, request),
-                DeleteIndexAction.NAME, "test user");
+            () -> authorize(authentication, DeleteIndexAction.NAME, request),
+            DeleteIndexAction.NAME, "test user");
         verify(auditTrail).accessDenied(authentication, DeleteIndexAction.NAME, request, Role.EMPTY.names());
         verifyNoMoreInteractions(auditTrail);
     }
 
     public void testUnknownRoleCausesDenial() {
         Tuple<String, TransportRequest> tuple = randomFrom(asList(
-                new Tuple<>(SearchAction.NAME, new SearchRequest()),
-                new Tuple<>(IndicesExistsAction.NAME, new IndicesExistsRequest()),
-                new Tuple<>(SqlQueryAction.NAME, new SqlQueryRequest())));
+            new Tuple<>(SearchAction.NAME, new SearchRequest()),
+            new Tuple<>(IndicesExistsAction.NAME, new IndicesExistsRequest()),
+            new Tuple<>(SqlQueryAction.NAME, new SqlQueryRequest())));
         String action = tuple.v1();
         TransportRequest request = tuple.v2();
         final Authentication authentication = createAuthentication(new User("test user", "non-existent-role"));
         mockEmptyMetaData();
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, action, request),
-                action, "test user");
+            () -> authorize(authentication, action, request),
+            action, "test user");
         verify(auditTrail).accessDenied(authentication, action, request, Role.EMPTY.names());
         verifyNoMoreInteractions(auditTrail);
     }
@@ -367,22 +427,22 @@ public class AuthorizationServiceTests extends ESTestCase {
         final TransportRequest request = mock(TransportRequest.class);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         final RoleDescriptor role = new RoleDescriptor("a_role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null);
         roleMap.put("a_all", role);
 
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, "whatever", request),
-                "whatever", "test user");
-        verify(auditTrail).accessDenied(authentication, "whatever", request, new String[] { role.getName() });
+            () -> authorize(authentication, "whatever", request),
+            "whatever", "test user");
+        verify(auditTrail).accessDenied(authentication, "whatever", request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
     public void testThatRoleWithNoIndicesIsDenied() {
         @SuppressWarnings("unchecked")
         Tuple<String, TransportRequest> tuple = randomFrom(
-                new Tuple<>(SearchAction.NAME, new SearchRequest()),
-                new Tuple<>(IndicesExistsAction.NAME, new IndicesExistsRequest()),
-                new Tuple<>(SqlQueryAction.NAME, new SqlQueryRequest()));
+            new Tuple<>(SearchAction.NAME, new SearchRequest()),
+            new Tuple<>(IndicesExistsAction.NAME, new IndicesExistsRequest()),
+            new Tuple<>(SqlQueryAction.NAME, new SqlQueryRequest()));
         String action = tuple.v1();
         TransportRequest request = tuple.v2();
         final Authentication authentication = createAuthentication(new User("test user", "no_indices"));
@@ -391,9 +451,9 @@ public class AuthorizationServiceTests extends ESTestCase {
         mockEmptyMetaData();
 
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, action, request),
-                action, "test user");
-        verify(auditTrail).accessDenied(authentication, action, request, new String[] { role.getName() });
+            () -> authorize(authentication, action, request),
+            action, "test user");
+        verify(auditTrail).accessDenied(authentication, action, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
@@ -402,12 +462,12 @@ public class AuthorizationServiceTests extends ESTestCase {
         final Tuple<String, TransportRequest> request = randomCompositeRequest();
         authorize(authentication, request.v1(), request.v2());
 
-        verify(auditTrail).accessGranted(authentication, request.v1(), request.v2(), new String[] { ElasticUser.ROLE_NAME });
+        verify(auditTrail).accessGranted(authentication, request.v1(), request.v2(), new String[]{ElasticUser.ROLE_NAME});
     }
 
     public void testSearchAgainstEmptyCluster() {
         RoleDescriptor role = new RoleDescriptor("a_role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         roleMap.put("a_all", role);
         mockEmptyMetaData();
@@ -415,25 +475,25 @@ public class AuthorizationServiceTests extends ESTestCase {
         {
             //ignore_unavailable set to false, user is not authorized for this index nor does it exist
             SearchRequest searchRequest = new SearchRequest("does_not_exist")
-                    .indicesOptions(IndicesOptions.fromOptions(false, true,
-                                                               true, false));
+                .indicesOptions(IndicesOptions.fromOptions(false, true,
+                    true, false));
 
             assertThrowsAuthorizationException(
-                    () -> authorize(authentication, SearchAction.NAME, searchRequest),
-                    SearchAction.NAME, "test user");
-            verify(auditTrail).accessDenied(authentication, SearchAction.NAME, searchRequest, new String[] { role.getName() });
+                () -> authorize(authentication, SearchAction.NAME, searchRequest),
+                SearchAction.NAME, "test user");
+            verify(auditTrail).accessDenied(authentication, SearchAction.NAME, searchRequest, new String[]{role.getName()});
             verifyNoMoreInteractions(auditTrail);
         }
 
         {
             //ignore_unavailable and allow_no_indices both set to true, user is not authorized for this index nor does it exist
             SearchRequest searchRequest = new SearchRequest("does_not_exist")
-                    .indicesOptions(IndicesOptions.fromOptions(true, true, true, false));
+                .indicesOptions(IndicesOptions.fromOptions(true, true, true, false));
             authorize(authentication, SearchAction.NAME, searchRequest);
-            verify(auditTrail).accessGranted(authentication, SearchAction.NAME, searchRequest, new String[] { role.getName() });
+            verify(auditTrail).accessGranted(authentication, SearchAction.NAME, searchRequest, new String[]{role.getName()});
             final IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
             final IndicesAccessControl.IndexAccessControl indexAccessControl =
-                    indicesAccessControl.getIndexPermissions(IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER);
+                indicesAccessControl.getIndexPermissions(IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER);
             assertFalse(indexAccessControl.getFieldPermissions().hasFieldLevelSecurity());
             assertNull(indexAccessControl.getQueries());
         }
@@ -441,40 +501,40 @@ public class AuthorizationServiceTests extends ESTestCase {
 
     public void testScrollRelatedRequestsAllowed() {
         RoleDescriptor role = new RoleDescriptor("a_role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         roleMap.put("a_all", role);
         mockEmptyMetaData();
 
         final ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         authorize(authentication, ClearScrollAction.NAME, clearScrollRequest);
-        verify(auditTrail).accessGranted(authentication, ClearScrollAction.NAME, clearScrollRequest, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, ClearScrollAction.NAME, clearScrollRequest, new String[]{role.getName()});
 
         final SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
         authorize(authentication, SearchScrollAction.NAME, searchScrollRequest);
-        verify(auditTrail).accessGranted(authentication, SearchScrollAction.NAME, searchScrollRequest, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, SearchScrollAction.NAME, searchScrollRequest, new String[]{role.getName()});
 
         // We have to use a mock request for other Scroll actions as the actual requests are package private to SearchTransportService
         final TransportRequest request = mock(TransportRequest.class);
         authorize(authentication, SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME, request);
         verify(auditTrail).accessGranted(authentication, SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME, request,
-                new String[] { role.getName() });
+            new String[]{role.getName()});
 
         authorize(authentication, SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME, request);
         verify(auditTrail).accessGranted(authentication, SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME, request,
-                new String[] { role.getName() });
+            new String[]{role.getName()});
 
         authorize(authentication, SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME, request);
         verify(auditTrail).accessGranted(authentication, SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME, request,
-                new String[] { role.getName() });
+            new String[]{role.getName()});
 
         authorize(authentication, SearchTransportService.QUERY_SCROLL_ACTION_NAME, request);
         verify(auditTrail).accessGranted(authentication, SearchTransportService.QUERY_SCROLL_ACTION_NAME, request,
-                new String[] { role.getName() });
+            new String[]{role.getName()});
 
         authorize(authentication, SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME, request);
         verify(auditTrail).accessGranted(authentication, SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME, request,
-                new String[] { role.getName() });
+            new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
@@ -482,14 +542,14 @@ public class AuthorizationServiceTests extends ESTestCase {
         TransportRequest request = new GetIndexRequest().indices("b");
         ClusterState state = mockEmptyMetaData();
         RoleDescriptor role = new RoleDescriptor("a_role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         roleMap.put("a_all", role);
 
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, "indices:a", request),
-                "indices:a", "test user");
-        verify(auditTrail).accessDenied(authentication, "indices:a", request, new String[] { role.getName() });
+            () -> authorize(authentication, "indices:a", request),
+            "indices:a", "test user");
+        verify(auditTrail).accessDenied(authentication, "indices:a", request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
         verify(clusterService, times(1)).state();
         verify(state, times(1)).metaData();
@@ -500,14 +560,14 @@ public class AuthorizationServiceTests extends ESTestCase {
         request.alias(new Alias("a2"));
         ClusterState state = mockEmptyMetaData();
         RoleDescriptor role = new RoleDescriptor("a_role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         roleMap.put("a_all", role);
 
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, CreateIndexAction.NAME, request),
-                IndicesAliasesAction.NAME, "test user");
-        verify(auditTrail).accessDenied(authentication, IndicesAliasesAction.NAME, request, new String[] { role.getName() });
+            () -> authorize(authentication, CreateIndexAction.NAME, request),
+            IndicesAliasesAction.NAME, "test user");
+        verify(auditTrail).accessDenied(authentication, IndicesAliasesAction.NAME, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
         verify(clusterService).state();
         verify(state, times(1)).metaData();
@@ -518,13 +578,13 @@ public class AuthorizationServiceTests extends ESTestCase {
         request.alias(new Alias("a2"));
         ClusterState state = mockEmptyMetaData();
         RoleDescriptor role = new RoleDescriptor("a_all", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a", "a2").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a", "a2").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         roleMap.put("a_all", role);
 
         authorize(authentication, CreateIndexAction.NAME, request);
 
-        verify(auditTrail).accessGranted(authentication, CreateIndexAction.NAME, request, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, CreateIndexAction.NAME, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
         verify(clusterService).state();
         verify(state, times(1)).metaData();
@@ -536,17 +596,17 @@ public class AuthorizationServiceTests extends ESTestCase {
         Settings settings = Settings.builder().put(AnonymousUser.ROLES_SETTING.getKey(), "a_all").build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         authorizationService = new AuthorizationService(settings, rolesStore, clusterService, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser);
+            new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser);
 
         RoleDescriptor role = new RoleDescriptor("a_all", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
         roleMap.put("a_all", role);
 
         final Authentication authentication = createAuthentication(anonymousUser);
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, "indices:a", request),
-                "indices:a", anonymousUser.principal());
-        verify(auditTrail).accessDenied(authentication, "indices:a", request, new String[] { role.getName() });
+            () -> authorize(authentication, "indices:a", request),
+            "indices:a", anonymousUser.principal());
+        verify(auditTrail).accessDenied(authentication, "indices:a", request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
         verify(clusterService, times(1)).state();
         verify(state, times(1)).metaData();
@@ -556,21 +616,21 @@ public class AuthorizationServiceTests extends ESTestCase {
         TransportRequest request = new GetIndexRequest().indices("b");
         ClusterState state = mockEmptyMetaData();
         Settings settings = Settings.builder()
-                .put(AnonymousUser.ROLES_SETTING.getKey(), "a_all")
-                .put(AuthorizationService.ANONYMOUS_AUTHORIZATION_EXCEPTION_SETTING.getKey(), false)
-                .build();
+            .put(AnonymousUser.ROLES_SETTING.getKey(), "a_all")
+            .put(AuthorizationService.ANONYMOUS_AUTHORIZATION_EXCEPTION_SETTING.getKey(), false)
+            .build();
         final Authentication authentication = createAuthentication(new AnonymousUser(settings));
         authorizationService = new AuthorizationService(settings, rolesStore, clusterService, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(settings));
+            new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(settings));
 
         RoleDescriptor role = new RoleDescriptor("a_all", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null);
         roleMap.put("a_all", role);
 
         final ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
-                () -> authorize(authentication, "indices:a", request));
+            () -> authorize(authentication, "indices:a", request));
         assertAuthenticationException(securityException, containsString("action [indices:a] requires authentication"));
-        verify(auditTrail).accessDenied(authentication, "indices:a", request, new String[] { role.getName() });
+        verify(auditTrail).accessDenied(authentication, "indices:a", request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
         verify(clusterService, times(1)).state();
         verify(state, times(1)).metaData();
@@ -581,16 +641,16 @@ public class AuthorizationServiceTests extends ESTestCase {
         TransportRequest request = new GetIndexRequest().indices("not-an-index-*").indicesOptions(options);
         ClusterState state = mockEmptyMetaData();
         RoleDescriptor role = new RoleDescriptor("a_all", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         roleMap.put("a_all", role);
 
         final IndexNotFoundException nfe = expectThrows(
-                IndexNotFoundException.class,
-                () -> authorize(authentication, GetIndexAction.NAME, request));
+            IndexNotFoundException.class,
+            () -> authorize(authentication, GetIndexAction.NAME, request));
         assertThat(nfe.getIndex(), is(notNullValue()));
         assertThat(nfe.getIndex().getName(), is("not-an-index-*"));
-        verify(auditTrail).accessDenied(authentication, GetIndexAction.NAME, request, new String[] { role.getName() });
+        verify(auditTrail).accessDenied(authentication, GetIndexAction.NAME, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
         verify(clusterService).state();
         verify(state, times(1)).metaData();
@@ -601,8 +661,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         final Authentication authentication = createAuthentication(new User("run as me", null, new User("test user", "admin")));
         assertNotEquals(authentication.getUser().authenticatedUser(), authentication);
         assertThrowsAuthorizationExceptionRunAs(
-                () -> authorize(authentication, "indices:a", request),
-                "indices:a", "test user", "run as me"); // run as [run as me]
+            () -> authorize(authentication, "indices:a", request),
+            "indices:a", "test user", "run as me"); // run as [run as me]
         verify(auditTrail).runAsDenied(authentication, "indices:a", request, Role.EMPTY.names());
         verifyNoMoreInteractions(auditTrail);
     }
@@ -610,67 +670,67 @@ public class AuthorizationServiceTests extends ESTestCase {
     public void testRunAsRequestWithoutLookedUpBy() {
         AuthenticateRequest request = new AuthenticateRequest("run as me");
         roleMap.put("can run as", ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR);
-        User user = new User("run as me", Strings.EMPTY_ARRAY, new User("test user", new String[] { "can run as" }));
+        User user = new User("run as me", Strings.EMPTY_ARRAY, new User("test user", new String[]{"can run as"}));
         Authentication authentication = new Authentication(user, new RealmRef("foo", "bar", "baz"), null);
         assertNotEquals(user.authenticatedUser(), user);
         assertThrowsAuthorizationExceptionRunAs(
-                () -> authorize(authentication, AuthenticateAction.NAME, request),
-                AuthenticateAction.NAME, "test user", "run as me"); // run as [run as me]
+            () -> authorize(authentication, AuthenticateAction.NAME, request),
+            AuthenticateAction.NAME, "test user", "run as me"); // run as [run as me]
         verify(auditTrail).runAsDenied(authentication, AuthenticateAction.NAME, request,
-                new String[] { ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName() });
+            new String[]{ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
     public void testRunAsRequestRunningAsUnAllowedUser() {
         TransportRequest request = mock(TransportRequest.class);
-        User user = new User("run as me", new String[] { "doesn't exist" }, new User("test user", "can run as"));
+        User user = new User("run as me", new String[]{"doesn't exist"}, new User("test user", "can run as"));
         assertNotEquals(user.authenticatedUser(), user);
         final Authentication authentication = createAuthentication(user);
         final RoleDescriptor role = new RoleDescriptor("can run as", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() },
-                new String[] { "not the right user" });
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()},
+            new String[]{"not the right user"});
         roleMap.put("can run as", role);
 
         assertThrowsAuthorizationExceptionRunAs(
-                () -> authorize(authentication, "indices:a", request),
-                "indices:a", "test user", "run as me");
-        verify(auditTrail).runAsDenied(authentication, "indices:a", request, new String[] { role.getName() });
+            () -> authorize(authentication, "indices:a", request),
+            "indices:a", "test user", "run as me");
+        verify(auditTrail).runAsDenied(authentication, "indices:a", request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
     public void testRunAsRequestWithRunAsUserWithoutPermission() {
         TransportRequest request = new GetIndexRequest().indices("a");
         User authenticatedUser = new User("test user", "can run as");
-        User user = new User("run as me", new String[] { "b" }, authenticatedUser);
+        User user = new User("run as me", new String[]{"b"}, authenticatedUser);
         assertNotEquals(user.authenticatedUser(), user);
         final Authentication authentication = createAuthentication(user);
         final RoleDescriptor runAsRole = new RoleDescriptor("can run as", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() },
-                new String[] { "run as me" });
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()},
+            new String[]{"run as me"});
         roleMap.put("can run as", runAsRole);
 
         RoleDescriptor bRole = new RoleDescriptor("b", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("b").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("b").privileges("all").build()}, null);
         boolean indexExists = randomBoolean();
         if (indexExists) {
             ClusterState state = mock(ClusterState.class);
             when(clusterService.state()).thenReturn(state);
             when(state.metaData()).thenReturn(MetaData.builder()
-                    .put(new IndexMetaData.Builder("a")
-                            .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
-                            .numberOfShards(1).numberOfReplicas(0).build(), true)
-                    .build());
+                .put(new IndexMetaData.Builder("a")
+                    .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
+                    .numberOfShards(1).numberOfReplicas(0).build(), true)
+                .build());
             roleMap.put("b", bRole);
         } else {
             mockEmptyMetaData();
         }
 
         assertThrowsAuthorizationExceptionRunAs(
-                () -> authorize(authentication, "indices:a", request),
-                "indices:a", "test user", "run as me");
-        verify(auditTrail).runAsGranted(authentication, "indices:a", request, new String[] { runAsRole.getName() });
+            () -> authorize(authentication, "indices:a", request),
+            "indices:a", "test user", "run as me");
+        verify(auditTrail).runAsGranted(authentication, "indices:a", request, new String[]{runAsRole.getName()});
         if (indexExists) {
-            verify(auditTrail).accessDenied(authentication, "indices:a", request, new String[] { bRole.getName() });
+            verify(auditTrail).accessDenied(authentication, "indices:a", request, new String[]{bRole.getName()});
         } else {
             verify(auditTrail).accessDenied(authentication, "indices:a", request, Role.EMPTY.names());
         }
@@ -679,43 +739,43 @@ public class AuthorizationServiceTests extends ESTestCase {
 
     public void testRunAsRequestWithValidPermissions() {
         TransportRequest request = new GetIndexRequest().indices("b");
-        User authenticatedUser = new User("test user", new String[] { "can run as" });
-        User user = new User("run as me", new String[] { "b" }, authenticatedUser);
+        User authenticatedUser = new User("test user", new String[]{"can run as"});
+        User user = new User("run as me", new String[]{"b"}, authenticatedUser);
         assertNotEquals(user.authenticatedUser(), user);
         final Authentication authentication = createAuthentication(user);
         final RoleDescriptor runAsRole = new RoleDescriptor("can run as", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() },
-                new String[] { "run as me" });
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()},
+            new String[]{"run as me"});
         roleMap.put("can run as", runAsRole);
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder("b")
-                        .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
-                        .numberOfShards(1).numberOfReplicas(0).build(), true)
-                .build());
+            .put(new IndexMetaData.Builder("b")
+                .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
+                .numberOfShards(1).numberOfReplicas(0).build(), true)
+            .build());
         RoleDescriptor bRole = new RoleDescriptor("b", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("b").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("b").privileges("all").build()}, null);
         roleMap.put("b", bRole);
 
         authorize(authentication, "indices:a", request);
-        verify(auditTrail).runAsGranted(authentication, "indices:a", request, new String[] { runAsRole.getName() });
-        verify(auditTrail).accessGranted(authentication, "indices:a", request, new String[] { bRole.getName() });
+        verify(auditTrail).runAsGranted(authentication, "indices:a", request, new String[]{runAsRole.getName()});
+        verify(auditTrail).accessGranted(authentication, "indices:a", request, new String[]{bRole.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
     public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() {
-        RoleDescriptor role = new RoleDescriptor("all access", new String[] { "all" },
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("*").privileges("all").build() }, null);
+        RoleDescriptor role = new RoleDescriptor("all access", new String[]{"all"},
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("*").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("all_access_user", "all_access"));
         roleMap.put("all_access", role);
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME)
-                        .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
-                        .numberOfShards(1).numberOfReplicas(0).build(), true)
-                .build());
+            .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME)
+                .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
+                .numberOfShards(1).numberOfReplicas(0).build(), true)
+            .build());
 
         List<Tuple<String, TransportRequest>> requests = new ArrayList<>();
         requests.add(new Tuple<>(BulkAction.NAME + "[s]",
@@ -726,34 +786,34 @@ public class AuthorizationServiceTests extends ESTestCase {
             new IndexRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id")));
+            new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id")));
+            new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest()
-                .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME))));
+            .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME))));
         requests.add(
-                new Tuple<>(UpdateSettingsAction.NAME, new UpdateSettingsRequest().indices(SECURITY_INDEX_NAME)));
+            new Tuple<>(UpdateSettingsAction.NAME, new UpdateSettingsRequest().indices(SECURITY_INDEX_NAME)));
 
         for (Tuple<String, TransportRequest> requestTuple : requests) {
             String action = requestTuple.v1();
             TransportRequest request = requestTuple.v2();
             assertThrowsAuthorizationException(
-                    () -> authorize(authentication, action, request),
-                    action, "all_access_user");
-            verify(auditTrail).accessDenied(authentication, action, request, new String[] { role.getName() });
+                () -> authorize(authentication, action, request),
+                action, "all_access_user");
+            verify(auditTrail).accessDenied(authentication, action, request, new String[]{role.getName()});
             verifyNoMoreInteractions(auditTrail);
         }
 
         // we should allow waiting for the health of the index or any index if the user has this permission
         ClusterHealthRequest request = new ClusterHealthRequest(SECURITY_INDEX_NAME);
         authorize(authentication, ClusterHealthAction.NAME, request);
-        verify(auditTrail).accessGranted(authentication, ClusterHealthAction.NAME, request, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, ClusterHealthAction.NAME, request, new String[]{role.getName()});
 
         // multiple indices
         request = new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar");
         authorize(authentication, ClusterHealthAction.NAME, request);
-        verify(auditTrail).accessGranted(authentication, ClusterHealthAction.NAME, request, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, ClusterHealthAction.NAME, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
 
         final SearchRequest searchRequest = new SearchRequest("_all");
@@ -763,17 +823,17 @@ public class AuthorizationServiceTests extends ESTestCase {
     }
 
     public void testGrantedNonXPackUserCanExecuteMonitoringOperationsAgainstSecurityIndex() {
-        RoleDescriptor role = new RoleDescriptor("all access", new String[] { "all" },
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("*").privileges("all").build() }, null);
+        RoleDescriptor role = new RoleDescriptor("all access", new String[]{"all"},
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("*").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("all_access_user", "all_access"));
         roleMap.put("all_access", role);
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME)
-                        .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
-                        .numberOfShards(1).numberOfReplicas(0).build(), true)
-                .build());
+            .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME)
+                .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
+                .numberOfShards(1).numberOfReplicas(0).build(), true)
+            .build());
 
         List<Tuple<String, ? extends TransportRequest>> requests = new ArrayList<>();
         requests.add(new Tuple<>(IndicesStatsAction.NAME, new IndicesStatsRequest().indices(SECURITY_INDEX_NAME)));
@@ -781,15 +841,15 @@ public class AuthorizationServiceTests extends ESTestCase {
         requests.add(new Tuple<>(IndicesSegmentsAction.NAME, new IndicesSegmentsRequest().indices(SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(IndicesShardStoresAction.NAME,
-                new IndicesShardStoresRequest().indices(SECURITY_INDEX_NAME)));
+            new IndicesShardStoresRequest().indices(SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(UpgradeStatusAction.NAME,
-                new UpgradeStatusRequest().indices(SECURITY_INDEX_NAME)));
+            new UpgradeStatusRequest().indices(SECURITY_INDEX_NAME)));
 
         for (final Tuple<String, ? extends TransportRequest> requestTuple : requests) {
             final String action = requestTuple.v1();
             final TransportRequest request = requestTuple.v2();
             authorize(authentication, action, request);
-            verify(auditTrail).accessGranted(authentication, action, request, new String[] { role.getName() });
+            verify(auditTrail).accessGranted(authentication, action, request, new String[]{role.getName()});
         }
     }
 
@@ -799,33 +859,33 @@ public class AuthorizationServiceTests extends ESTestCase {
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME)
-                        .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
-                        .numberOfShards(1).numberOfReplicas(0).build(), true)
-                .build());
+            .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME)
+                .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
+                .numberOfShards(1).numberOfReplicas(0).build(), true)
+            .build());
 
         List<Tuple<String, TransportRequest>> requests = new ArrayList<>();
         requests.add(new Tuple<>(DeleteAction.NAME,
             new DeleteRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(BulkAction.NAME + "[s]",
-                createBulkShardRequest(SECURITY_INDEX_NAME, DeleteRequest::new)));
+            createBulkShardRequest(SECURITY_INDEX_NAME, DeleteRequest::new)));
         requests.add(new Tuple<>(UpdateAction.NAME,
             new UpdateRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(IndexAction.NAME,
             new IndexRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(BulkAction.NAME + "[s]",
-                createBulkShardRequest(SECURITY_INDEX_NAME, IndexRequest::new)));
+            createBulkShardRequest(SECURITY_INDEX_NAME, IndexRequest::new)));
         requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id")));
+            new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id")));
+            new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest()
-                .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME))));
+            .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME))));
         requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(ClusterHealthAction.NAME,
-                new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar")));
+            new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar")));
 
         for (final Tuple<String, TransportRequest> requestTuple : requests) {
             final String action = requestTuple.v1();
@@ -843,10 +903,10 @@ public class AuthorizationServiceTests extends ESTestCase {
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME)
-                        .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
-                        .numberOfShards(1).numberOfReplicas(0).build(), true)
-                .build());
+            .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME)
+                .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
+                .numberOfShards(1).numberOfReplicas(0).build(), true)
+            .build());
 
         String action = SearchAction.NAME;
         SearchRequest request = new SearchRequest("_all");
@@ -860,9 +920,9 @@ public class AuthorizationServiceTests extends ESTestCase {
         Settings settings = Settings.builder().put(AnonymousUser.ROLES_SETTING.getKey(), "anonymous_user_role").build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         authorizationService = new AuthorizationService(settings, rolesStore, clusterService, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser);
-        roleMap.put("anonymous_user_role", new RoleDescriptor("anonymous_user_role", new String[] { "all" },
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null));
+            new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser);
+        roleMap.put("anonymous_user_role", new RoleDescriptor("anonymous_user_role", new String[]{"all"},
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null));
         mockEmptyMetaData();
 
         // sanity check the anonymous user
@@ -886,9 +946,9 @@ public class AuthorizationServiceTests extends ESTestCase {
         Settings settings = Settings.builder().put(AnonymousUser.ROLES_SETTING.getKey(), "anonymous_user_role").build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         authorizationService = new AuthorizationService(settings, rolesStore, clusterService, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser);
-        roleMap.put("anonymous_user_role", new RoleDescriptor("anonymous_user_role", new String[] { "all" },
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null));
+            new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser);
+        roleMap.put("anonymous_user_role", new RoleDescriptor("anonymous_user_role", new String[]{"all"},
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null));
         mockEmptyMetaData();
         PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
         authorizationService.roles(new User("no role user"), rolesFuture);
@@ -905,8 +965,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         final RoleDescriptor role = new RoleDescriptor("no_indices", null, null, null);
         roleMap.put("no_indices", role);
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, action, request), action, "test user");
-        verify(auditTrail).accessDenied(authentication, action, request, new String[] { role.getName() });
+            () -> authorize(authentication, action, request), action, "test user");
+        verify(auditTrail).accessDenied(authentication, action, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
@@ -917,11 +977,11 @@ public class AuthorizationServiceTests extends ESTestCase {
         final TransportRequest request = compositeRequest.v2();
         final Authentication authentication = createAuthentication(new User("test user", "role"));
         final RoleDescriptor role = new RoleDescriptor("role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices(randomBoolean() ? "a" : "index").privileges("all").build() },
-                null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices(randomBoolean() ? "a" : "index").privileges("all").build()},
+            null);
         roleMap.put("role", role);
         authorize(authentication, action, request);
-        verify(auditTrail).accessGranted(authentication, action, request, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, action, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
@@ -930,10 +990,10 @@ public class AuthorizationServiceTests extends ESTestCase {
         TransportRequest request = mock(TransportRequest.class);
         User user = new User("test user", "role");
         roleMap.put("role", new RoleDescriptor("role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices(randomBoolean() ? "a" : "index").privileges("all").build() },
-                null));
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices(randomBoolean() ? "a" : "index").privileges("all").build()},
+            null));
         IllegalStateException illegalStateException = expectThrows(IllegalStateException.class,
-                () -> authorize(createAuthentication(user), action, request));
+            () -> authorize(createAuthentication(user), action, request));
         assertThat(illegalStateException.getMessage(), containsString("Composite actions must implement CompositeIndicesRequest"));
     }
 
@@ -970,62 +1030,62 @@ public class AuthorizationServiceTests extends ESTestCase {
 
         User userAllowed = new User("userAllowed", "roleAllowed");
         roleMap.put("roleAllowed", new RoleDescriptor("roleAllowed", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("index").privileges("all").build() }, null));
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("index").privileges("all").build()}, null));
         User userDenied = new User("userDenied", "roleDenied");
         roleMap.put("roleDenied", new RoleDescriptor("roleDenied", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null));
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null));
         mockEmptyMetaData();
         authorize(createAuthentication(userAllowed), action, request);
         assertThrowsAuthorizationException(
-                () -> authorize(createAuthentication(userDenied), action, request), action, "userDenied");
+            () -> authorize(createAuthentication(userDenied), action, request), action, "userDenied");
     }
 
     public void testAuthorizationOfIndividualBulkItems() {
         final String action = BulkAction.NAME + "[s]";
         final BulkItemRequest[] items = {
-                new BulkItemRequest(1, new DeleteRequest("concrete-index", "doc", "c1")),
-                new BulkItemRequest(2, new IndexRequest("concrete-index", "doc", "c2")),
-                new BulkItemRequest(3, new DeleteRequest("alias-1", "doc", "a1a")),
-                new BulkItemRequest(4, new IndexRequest("alias-1", "doc", "a1b")),
-                new BulkItemRequest(5, new DeleteRequest("alias-2", "doc", "a2a")),
-                new BulkItemRequest(6, new IndexRequest("alias-2", "doc", "a2b"))
+            new BulkItemRequest(1, new DeleteRequest("concrete-index", "doc", "c1")),
+            new BulkItemRequest(2, new IndexRequest("concrete-index", "doc", "c2")),
+            new BulkItemRequest(3, new DeleteRequest("alias-1", "doc", "a1a")),
+            new BulkItemRequest(4, new IndexRequest("alias-1", "doc", "a1b")),
+            new BulkItemRequest(5, new DeleteRequest("alias-2", "doc", "a2a")),
+            new BulkItemRequest(6, new IndexRequest("alias-2", "doc", "a2b"))
         };
         final ShardId shardId = new ShardId("concrete-index", UUID.randomUUID().toString(), 1);
         final TransportRequest request = new BulkShardRequest(shardId, WriteRequest.RefreshPolicy.IMMEDIATE, items);
 
         final Authentication authentication = createAuthentication(new User("user", "my-role"));
-        RoleDescriptor role = new RoleDescriptor("my-role", null, new IndicesPrivileges[] {
-                IndicesPrivileges.builder().indices("concrete-index").privileges("all").build(),
-                IndicesPrivileges.builder().indices("alias-1").privileges("index").build(),
-                IndicesPrivileges.builder().indices("alias-2").privileges("delete").build()
+        RoleDescriptor role = new RoleDescriptor("my-role", null, new IndicesPrivileges[]{
+            IndicesPrivileges.builder().indices("concrete-index").privileges("all").build(),
+            IndicesPrivileges.builder().indices("alias-1").privileges("index").build(),
+            IndicesPrivileges.builder().indices("alias-2").privileges("delete").build()
         }, null);
         roleMap.put("my-role", role);
 
         mockEmptyMetaData();
         authorize(authentication, action, request);
 
-        verify(auditTrail).accessDenied(authentication, DeleteAction.NAME, request, new String[] { role.getName() }); // alias-1 delete
-        verify(auditTrail).accessDenied(authentication, IndexAction.NAME, request, new String[] { role.getName() }); // alias-2 index
-        verify(auditTrail).accessGranted(authentication, action, request, new String[] { role.getName() }); // bulk request is allowed
+        verify(auditTrail).accessDenied(authentication, DeleteAction.NAME, request, new String[]{role.getName()}); // alias-1 delete
+        verify(auditTrail).accessDenied(authentication, IndexAction.NAME, request, new String[]{role.getName()}); // alias-2 index
+        verify(auditTrail).accessGranted(authentication, action, request, new String[]{role.getName()}); // bulk request is allowed
         verifyNoMoreInteractions(auditTrail);
     }
 
     public void testAuthorizationOfIndividualBulkItemsWithDateMath() {
         final String action = BulkAction.NAME + "[s]";
         final BulkItemRequest[] items = {
-                new BulkItemRequest(1, new IndexRequest("<datemath-{now/M{YYYY}}>", "doc", "dy1")),
-                new BulkItemRequest(2,
-                    new DeleteRequest("<datemath-{now/d{YYYY}}>", "doc", "dy2")), // resolves to same as above
-                new BulkItemRequest(3, new IndexRequest("<datemath-{now/M{YYYY.MM}}>", "doc", "dm1")),
-                new BulkItemRequest(4,
-                    new DeleteRequest("<datemath-{now/d{YYYY.MM}}>", "doc", "dm2")), // resolves to same as above
+            new BulkItemRequest(1, new IndexRequest("<datemath-{now/M{YYYY}}>", "doc", "dy1")),
+            new BulkItemRequest(2,
+                new DeleteRequest("<datemath-{now/d{YYYY}}>", "doc", "dy2")), // resolves to same as above
+            new BulkItemRequest(3, new IndexRequest("<datemath-{now/M{YYYY.MM}}>", "doc", "dm1")),
+            new BulkItemRequest(4,
+                new DeleteRequest("<datemath-{now/d{YYYY.MM}}>", "doc", "dm2")), // resolves to same as above
         };
         final ShardId shardId = new ShardId("concrete-index", UUID.randomUUID().toString(), 1);
         final TransportRequest request = new BulkShardRequest(shardId, WriteRequest.RefreshPolicy.IMMEDIATE, items);
 
         final Authentication authentication = createAuthentication(new User("user", "my-role"));
         final RoleDescriptor role = new RoleDescriptor("my-role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("datemath-*").privileges("index").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("datemath-*").privileges("index").build()}, null);
         roleMap.put("my-role", role);
 
         mockEmptyMetaData();
@@ -1033,14 +1093,14 @@ public class AuthorizationServiceTests extends ESTestCase {
 
         // both deletes should fail
         verify(auditTrail, Mockito.times(2)).accessDenied(authentication, DeleteAction.NAME, request,
-            new String[] { role.getName() });
+            new String[]{role.getName()});
         // bulk request is allowed
-        verify(auditTrail).accessGranted(authentication, action, request, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, action, request, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
     private BulkShardRequest createBulkShardRequest(String indexName, TriFunction<String, String, String, DocWriteRequest<?>> req) {
-        final BulkItemRequest[] items = { new BulkItemRequest(1, req.apply(indexName, "type", "id")) };
+        final BulkItemRequest[] items = {new BulkItemRequest(1, req.apply(indexName, "type", "id"))};
         return new BulkShardRequest(new ShardId(indexName, UUID.randomUUID().toString(), 1),
             WriteRequest.RefreshPolicy.IMMEDIATE, items);
     }
@@ -1049,37 +1109,37 @@ public class AuthorizationServiceTests extends ESTestCase {
         final User user = new User("joe");
         final boolean changePasswordRequest = randomBoolean();
         final TransportRequest request = changePasswordRequest ?
-                new ChangePasswordRequestBuilder(mock(Client.class)).username(user.principal()).request() :
-                new AuthenticateRequestBuilder(mock(Client.class)).username(user.principal()).request();
+            new ChangePasswordRequestBuilder(mock(Client.class)).username(user.principal()).request() :
+            new AuthenticateRequestBuilder(mock(Client.class)).username(user.principal()).request();
         final String action = changePasswordRequest ? ChangePasswordAction.NAME : AuthenticateAction.NAME;
         final Authentication authentication = mock(Authentication.class);
         final RealmRef authenticatedBy = mock(RealmRef.class);
         when(authentication.getUser()).thenReturn(user);
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
         when(authenticatedBy.getType())
-                .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
-                    randomAlphaOfLengthBetween(4, 12));
+            .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
+                randomAlphaOfLengthBetween(4, 12));
 
         assertThat(request, instanceOf(UserRequest.class));
         assertTrue(AuthorizationService.checkSameUserPermissions(action, request, authentication));
     }
 
     public void testSameUserPermissionDoesNotAllowNonMatchingUsername() {
-        final User authUser = new User("admin", new String[] { "bar" });
+        final User authUser = new User("admin", new String[]{"bar"});
         final User user = new User("joe", null, authUser);
         final boolean changePasswordRequest = randomBoolean();
         final String username = randomFrom("", "joe" + randomAlphaOfLengthBetween(1, 5), randomAlphaOfLengthBetween(3, 10));
         final TransportRequest request = changePasswordRequest ?
-                new ChangePasswordRequestBuilder(mock(Client.class)).username(username).request() :
-                new AuthenticateRequestBuilder(mock(Client.class)).username(username).request();
+            new ChangePasswordRequestBuilder(mock(Client.class)).username(username).request() :
+            new AuthenticateRequestBuilder(mock(Client.class)).username(username).request();
         final String action = changePasswordRequest ? ChangePasswordAction.NAME : AuthenticateAction.NAME;
         final Authentication authentication = mock(Authentication.class);
         final RealmRef authenticatedBy = mock(RealmRef.class);
         when(authentication.getUser()).thenReturn(user);
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
         when(authenticatedBy.getType())
-                .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
-                    randomAlphaOfLengthBetween(4, 12));
+            .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
+                randomAlphaOfLengthBetween(4, 12));
 
         assertThat(request, instanceOf(UserRequest.class));
         assertFalse(AuthorizationService.checkSameUserPermissions(action, request, authentication));
@@ -1088,8 +1148,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         final RealmRef lookedUpBy = mock(RealmRef.class);
         when(authentication.getLookedUpBy()).thenReturn(lookedUpBy);
         when(lookedUpBy.getType())
-                .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
-                    randomAlphaOfLengthBetween(4, 12));
+            .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
+                randomAlphaOfLengthBetween(4, 12));
         // this should still fail since the username is still different
         assertFalse(AuthorizationService.checkSameUserPermissions(action, request, authentication));
 
@@ -1105,7 +1165,7 @@ public class AuthorizationServiceTests extends ESTestCase {
         final User user = mock(User.class);
         final TransportRequest request = mock(TransportRequest.class);
         final String action = randomFrom(PutUserAction.NAME, DeleteUserAction.NAME, ClusterHealthAction.NAME, ClusterStateAction.NAME,
-                ClusterStatsAction.NAME, GetLicenseAction.NAME);
+            ClusterStatsAction.NAME, GetLicenseAction.NAME);
         final Authentication authentication = mock(Authentication.class);
         final RealmRef authenticatedBy = mock(RealmRef.class);
         final boolean runAs = randomBoolean();
@@ -1114,20 +1174,20 @@ public class AuthorizationServiceTests extends ESTestCase {
         when(user.isRunAs()).thenReturn(runAs);
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
         when(authenticatedBy.getType())
-                .thenReturn(randomAlphaOfLengthBetween(4, 12));
+            .thenReturn(randomAlphaOfLengthBetween(4, 12));
 
         assertFalse(AuthorizationService.checkSameUserPermissions(action, request, authentication));
         verifyZeroInteractions(user, request, authentication);
     }
 
     public void testSameUserPermissionRunAsChecksAuthenticatedBy() {
-        final User authUser = new User("admin", new String[] { "bar" });
+        final User authUser = new User("admin", new String[]{"bar"});
         final String username = "joe";
         final User user = new User(username, null, authUser);
         final boolean changePasswordRequest = randomBoolean();
         final TransportRequest request = changePasswordRequest ?
-                new ChangePasswordRequestBuilder(mock(Client.class)).username(username).request() :
-                new AuthenticateRequestBuilder(mock(Client.class)).username(username).request();
+            new ChangePasswordRequestBuilder(mock(Client.class)).username(username).request() :
+            new AuthenticateRequestBuilder(mock(Client.class)).username(username).request();
         final String action = changePasswordRequest ? ChangePasswordAction.NAME : AuthenticateAction.NAME;
         final Authentication authentication = mock(Authentication.class);
         final RealmRef authenticatedBy = mock(RealmRef.class);
@@ -1136,8 +1196,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
         when(authentication.getLookedUpBy()).thenReturn(lookedUpBy);
         when(lookedUpBy.getType())
-                .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
-                    randomAlphaOfLengthBetween(4, 12));
+            .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
+                randomAlphaOfLengthBetween(4, 12));
         assertTrue(AuthorizationService.checkSameUserPermissions(action, request, authentication));
 
         when(authentication.getUser()).thenReturn(authUser);
@@ -1153,8 +1213,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         when(authentication.getUser()).thenReturn(user);
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
         when(authenticatedBy.getType()).thenReturn(randomFrom(LdapRealmSettings.LDAP_TYPE, FileRealmSettings.TYPE,
-                LdapRealmSettings.AD_TYPE, PkiRealmSettings.TYPE,
-                randomAlphaOfLengthBetween(4, 12)));
+            LdapRealmSettings.AD_TYPE, PkiRealmSettings.TYPE,
+            randomAlphaOfLengthBetween(4, 12)));
 
         assertThat(request, instanceOf(UserRequest.class));
         assertFalse(AuthorizationService.checkSameUserPermissions(action, request, authentication));
@@ -1165,7 +1225,7 @@ public class AuthorizationServiceTests extends ESTestCase {
     }
 
     public void testSameUserPermissionDoesNotAllowChangePasswordForLookedUpByOtherRealms() {
-        final User authUser = new User("admin", new String[] { "bar" });
+        final User authUser = new User("admin", new String[]{"bar"});
         final User user = new User("joe", null, authUser);
         final ChangePasswordRequest request = new ChangePasswordRequestBuilder(mock(Client.class)).username(user.principal()).request();
         final String action = ChangePasswordAction.NAME;
@@ -1176,8 +1236,8 @@ public class AuthorizationServiceTests extends ESTestCase {
         when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
         when(authentication.getLookedUpBy()).thenReturn(lookedUpBy);
         when(lookedUpBy.getType()).thenReturn(randomFrom(LdapRealmSettings.LDAP_TYPE, FileRealmSettings.TYPE,
-                LdapRealmSettings.AD_TYPE, PkiRealmSettings.TYPE,
-                randomAlphaOfLengthBetween(4, 12)));
+            LdapRealmSettings.AD_TYPE, PkiRealmSettings.TYPE,
+            randomAlphaOfLengthBetween(4, 12)));
 
         assertThat(request, instanceOf(UserRequest.class));
         assertFalse(AuthorizationService.checkSameUserPermissions(action, request, authentication));
@@ -1223,7 +1283,7 @@ public class AuthorizationServiceTests extends ESTestCase {
 
     public void testGetRolesForSystemUserThrowsException() {
         IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> authorizationService.roles(SystemUser.INSTANCE,
-                null));
+            null));
         assertEquals("the user [_system] is the system user and we should never try to get its roles", iae.getMessage());
     }
 
@@ -1245,9 +1305,9 @@ public class AuthorizationServiceTests extends ESTestCase {
         TransportRequest transportRequest = TransportActionProxy.wrapRequest(node, request);
         User user = new User("test user", "role");
         IllegalStateException illegalStateException = expectThrows(IllegalStateException.class,
-                () -> authorize(createAuthentication(user), "indices:some/action", transportRequest));
+            () -> authorize(createAuthentication(user), "indices:some/action", transportRequest));
         assertThat(illegalStateException.getMessage(),
-                startsWith("originalRequest is a proxy request for: [org.elasticsearch.transport.TransportRequest$"));
+            startsWith("originalRequest is a proxy request for: [org.elasticsearch.transport.TransportRequest$"));
         assertThat(illegalStateException.getMessage(), endsWith("] but action: [indices:some/action] isn't"));
     }
 
@@ -1255,11 +1315,11 @@ public class AuthorizationServiceTests extends ESTestCase {
         TransportRequest request = TransportRequest.Empty.INSTANCE;
         User user = new User("test user", "role");
         IllegalStateException illegalStateException = expectThrows(IllegalStateException.class,
-                () -> authorize(createAuthentication(user), TransportActionProxy.getProxyAction("indices:some/action"), request));
+            () -> authorize(createAuthentication(user), TransportActionProxy.getProxyAction("indices:some/action"), request));
         assertThat(illegalStateException.getMessage(),
-                startsWith("originalRequest is not a proxy request: [org.elasticsearch.transport.TransportRequest$"));
+            startsWith("originalRequest is not a proxy request: [org.elasticsearch.transport.TransportRequest$"));
         assertThat(illegalStateException.getMessage(),
-                endsWith("] but action: [internal:transport/proxy/indices:some/action] is a proxy action"));
+            endsWith("] but action: [internal:transport/proxy/indices:some/action] is a proxy action"));
     }
 
     public void testProxyRequestAuthenticationDenied() {
@@ -1271,14 +1331,14 @@ public class AuthorizationServiceTests extends ESTestCase {
         final RoleDescriptor role = new RoleDescriptor("no_indices", null, null, null);
         roleMap.put("no_indices", role);
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, action, transportRequest), action, "test user");
-        verify(auditTrail).accessDenied(authentication, action, proxiedRequest, new String[] { role.getName() });
+            () -> authorize(authentication, action, transportRequest), action, "test user");
+        verify(auditTrail).accessDenied(authentication, action, proxiedRequest, new String[]{role.getName()});
         verifyNoMoreInteractions(auditTrail);
     }
 
     public void testProxyRequestAuthenticationGrantedWithAllPrivileges() {
         RoleDescriptor role = new RoleDescriptor("a_role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("all").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("all").build()}, null);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         roleMap.put("a_all", role);
         mockEmptyMetaData();
@@ -1288,12 +1348,12 @@ public class AuthorizationServiceTests extends ESTestCase {
         final TransportRequest transportRequest = TransportActionProxy.wrapRequest(node, clearScrollRequest);
         final String action = TransportActionProxy.getProxyAction(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME);
         authorize(authentication, action, transportRequest);
-        verify(auditTrail).accessGranted(authentication, action, clearScrollRequest, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, action, clearScrollRequest, new String[]{role.getName()});
     }
 
     public void testProxyRequestAuthenticationGranted() {
         RoleDescriptor role = new RoleDescriptor("a_role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("read_cross_cluster").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("read_cross_cluster").build()}, null);
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         roleMap.put("a_all", role);
         mockEmptyMetaData();
@@ -1303,13 +1363,13 @@ public class AuthorizationServiceTests extends ESTestCase {
         final TransportRequest transportRequest = TransportActionProxy.wrapRequest(node, clearScrollRequest);
         final String action = TransportActionProxy.getProxyAction(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME);
         authorize(authentication, action, transportRequest);
-        verify(auditTrail).accessGranted(authentication, action, clearScrollRequest, new String[] { role.getName() });
+        verify(auditTrail).accessGranted(authentication, action, clearScrollRequest, new String[]{role.getName()});
     }
 
     public void testProxyRequestAuthenticationDeniedWithReadPrivileges() {
         final Authentication authentication = createAuthentication(new User("test user", "a_all"));
         final RoleDescriptor role = new RoleDescriptor("a_role", null,
-                new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a").privileges("read").build() }, null);
+            new IndicesPrivileges[]{IndicesPrivileges.builder().indices("a").privileges("read").build()}, null);
         roleMap.put("a_all", role);
         mockEmptyMetaData();
         DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT);
@@ -1317,7 +1377,7 @@ public class AuthorizationServiceTests extends ESTestCase {
         TransportRequest transportRequest = TransportActionProxy.wrapRequest(node, clearScrollRequest);
         String action = TransportActionProxy.getProxyAction(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME);
         assertThrowsAuthorizationException(
-                () -> authorize(authentication, action, transportRequest), action, "test user");
-        verify(auditTrail).accessDenied(authentication, action, clearScrollRequest, new String[] { role.getName() });
+            () -> authorize(authentication, action, transportRequest), action, "test user");
+        verify(auditTrail).accessDenied(authentication, action, clearScrollRequest, new String[]{role.getName()});
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
index 1d0e5c179a9..c48ac456898 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.authz;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -24,6 +25,7 @@ import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.util.List;
+import java.util.Set;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 
@@ -57,8 +59,10 @@ public class AuthorizedIndicesTests extends ESTestCase {
                         .putAlias(new AliasMetaData.Builder("ba").build())
                         .build(), true)
                 .build();
-        Role roles = CompositeRolesStore.buildRoleFromDescriptors(Sets.newHashSet(aStarRole, bRole),
-                new FieldPermissionsCache(Settings.EMPTY));
+        final PlainActionFuture<Role> future = new PlainActionFuture<>();
+        final Set<RoleDescriptor> descriptors = Sets.newHashSet(aStarRole, bRole);
+        CompositeRolesStore.buildRoleFromDescriptors(descriptors, new FieldPermissionsCache(Settings.EMPTY), null, future);
+        Role roles = future.actionGet();
         AuthorizedIndices authorizedIndices = new AuthorizedIndices(user, roles, SearchAction.NAME, metaData);
         List<String> list = authorizedIndices.get();
         assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
index bd5acdec818..eed3297661a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
@@ -174,8 +174,9 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
                 if (roleDescriptors.isEmpty()) {
                     callback.onResponse(Role.EMPTY);
                 } else {
-                    callback.onResponse(
-                            CompositeRolesStore.buildRoleFromDescriptors(roleDescriptors, fieldPermissionsCache));
+                    CompositeRolesStore.buildRoleFromDescriptors(roleDescriptors, fieldPermissionsCache, null,
+                            ActionListener.wrap(r -> callback.onResponse(r), callback::onFailure)
+                    );
                 }
                 return Void.TYPE;
             }).when(rolesStore).roles(any(Set.class), any(FieldPermissionsCache.class), any(ActionListener.class));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java
index 9d34382d566..07686838ad0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java
@@ -10,18 +10,34 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
+import org.hamcrest.Matchers;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.emptyArray;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.Is.is;
 
 public class RoleDescriptorTests extends ESTestCase {
@@ -45,9 +61,26 @@ public class RoleDescriptorTests extends ESTestCase {
                         .query("{\"query\": {\"match_all\": {}}}")
                         .build()
         };
-        RoleDescriptor descriptor = new RoleDescriptor("test", new String[] { "all", "none" }, groups, new String[] { "sudo" });
-        assertThat(descriptor.toString(), is("Role[name=test, cluster=[all,none], indicesPrivileges=[IndicesPrivileges[indices=[i1,i2], " +
-                "privileges=[read], field_security=[grant=[body,title], except=null], query={\"query\": {\"match_all\": {}}}],]" +
+        final RoleDescriptor.ApplicationResourcePrivileges[] applicationPrivileges = {
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .application("my_app")
+                .privileges("read", "write")
+                .resources("*")
+                .build()
+        };
+
+        final ConditionalClusterPrivilege[] conditionalClusterPrivileges = new ConditionalClusterPrivilege[]{
+            new ConditionalClusterPrivileges.ManageApplicationPrivileges(new LinkedHashSet<>(Arrays.asList("app01", "app02")))
+        };
+
+        RoleDescriptor descriptor = new RoleDescriptor("test", new String[] { "all", "none" }, groups, applicationPrivileges,
+            conditionalClusterPrivileges, new String[] { "sudo" }, Collections.emptyMap(), Collections.emptyMap());
+
+        assertThat(descriptor.toString(), is("Role[name=test, cluster=[all,none]" +
+                ", global=[{APPLICATION:manage:applications=app01,app02}]" +
+                ", indicesPrivileges=[IndicesPrivileges[indices=[i1,i2], privileges=[read]" +
+                ", field_security=[grant=[body,title], except=null], query={\"query\": {\"match_all\": {}}}],]" +
+                ", applicationPrivileges=[ApplicationResourcePrivileges[application=my_app, privileges=[read,write], resources=[*]],]" +
                 ", runAs=[sudo], metadata=[{}]]"));
     }
 
@@ -60,11 +93,23 @@ public class RoleDescriptorTests extends ESTestCase {
                         .query("{\"query\": {\"match_all\": {}}}")
                         .build()
         };
+        final RoleDescriptor.ApplicationResourcePrivileges[] applicationPrivileges = {
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .application("my_app")
+                .privileges("read", "write")
+                .resources("*")
+                .build()
+        };
+        final ConditionalClusterPrivilege[] conditionalClusterPrivileges = {
+            new ConditionalClusterPrivileges.ManageApplicationPrivileges(new LinkedHashSet<>(Arrays.asList("app01", "app02")))
+        };
+
         Map<String, Object> metadata = randomBoolean() ? MetadataUtils.DEFAULT_RESERVED_METADATA : null;
-        RoleDescriptor descriptor = new RoleDescriptor("test", new String[] { "all", "none" }, groups, new String[] { "sudo" }, metadata);
+        RoleDescriptor descriptor = new RoleDescriptor("test", new String[] { "all", "none" }, groups, applicationPrivileges,
+            conditionalClusterPrivileges, new String[]{ "sudo" }, metadata, Collections.emptyMap());
         XContentBuilder builder = descriptor.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS);
         RoleDescriptor parsed = RoleDescriptor.parse("test", BytesReference.bytes(builder), false, XContentType.JSON);
-        assertEquals(parsed, descriptor);
+        assertThat(parsed, equalTo(descriptor));
     }
 
     public void testParse() throws Exception {
@@ -113,6 +158,53 @@ public class RoleDescriptorTests extends ESTestCase {
         assertNotNull(rd.getMetadata());
         assertThat(rd.getMetadata().size(), is(1));
         assertThat(rd.getMetadata().get("foo"), is("bar"));
+
+        q = "{\"cluster\":[\"a\", \"b\"], \"run_as\": [\"m\", \"n\"]," +
+                " \"index\": [{\"names\": [\"idx1\",\"idx2\"], \"privileges\": [\"p1\", \"p2\"]}]," +
+                " \"applications\": [" +
+                "     {\"resources\": [\"object-123\",\"object-456\"], \"privileges\":[\"read\", \"delete\"], \"application\":\"app1\"}," +
+                "     {\"resources\": [\"*\"], \"privileges\":[\"admin\"], \"application\":\"app2\" }" +
+                " ]," +
+                " \"global\": { \"application\": { \"manage\": { \"applications\" : [ \"kibana\", \"logstash\" ] } } }" +
+                "}";
+        rd = RoleDescriptor.parse("test", new BytesArray(q), false, XContentType.JSON);
+        assertThat(rd.getName(), equalTo("test"));
+        assertThat(rd.getClusterPrivileges(), arrayContaining("a", "b"));
+        assertThat(rd.getIndicesPrivileges().length, equalTo(1));
+        assertThat(rd.getIndicesPrivileges()[0].getIndices(), arrayContaining("idx1", "idx2"));
+        assertThat(rd.getRunAs(), arrayContaining("m", "n"));
+        assertThat(rd.getIndicesPrivileges()[0].getQuery(), nullValue());
+        assertThat(rd.getApplicationPrivileges().length, equalTo(2));
+        assertThat(rd.getApplicationPrivileges()[0].getResources(), arrayContaining("object-123", "object-456"));
+        assertThat(rd.getApplicationPrivileges()[0].getPrivileges(), arrayContaining("read", "delete"));
+        assertThat(rd.getApplicationPrivileges()[0].getApplication(), equalTo("app1"));
+        assertThat(rd.getApplicationPrivileges()[1].getResources(), arrayContaining("*"));
+        assertThat(rd.getApplicationPrivileges()[1].getPrivileges(), arrayContaining("admin"));
+        assertThat(rd.getApplicationPrivileges()[1].getApplication(), equalTo("app2"));
+        assertThat(rd.getConditionalClusterPrivileges(), Matchers.arrayWithSize(1));
+
+        final ConditionalClusterPrivilege conditionalPrivilege = rd.getConditionalClusterPrivileges()[0];
+        assertThat(conditionalPrivilege.getCategory(), equalTo(ConditionalClusterPrivilege.Category.APPLICATION));
+        assertThat(conditionalPrivilege, instanceOf(ConditionalClusterPrivileges.ManageApplicationPrivileges.class));
+        assertThat(((ConditionalClusterPrivileges.ManageApplicationPrivileges) conditionalPrivilege).getApplicationNames(),
+            containsInAnyOrder("kibana", "logstash"));
+
+        q = "{\"applications\": [{\"application\": \"myapp\", \"resources\": [\"*\"], \"privileges\": [\"login\" ]}] }";
+        rd = RoleDescriptor.parse("test", new BytesArray(q), false, XContentType.JSON);
+        assertThat(rd.getName(), equalTo("test"));
+        assertThat(rd.getClusterPrivileges(), emptyArray());
+        assertThat(rd.getIndicesPrivileges(), emptyArray());
+        assertThat(rd.getApplicationPrivileges().length, equalTo(1));
+        assertThat(rd.getApplicationPrivileges()[0].getResources(), arrayContaining("*"));
+        assertThat(rd.getApplicationPrivileges()[0].getPrivileges(), arrayContaining("login"));
+        assertThat(rd.getApplicationPrivileges()[0].getApplication(), equalTo("myapp"));
+        assertThat(rd.getConditionalClusterPrivileges(), Matchers.arrayWithSize(0));
+
+        final String badJson
+                = "{\"applications\":[{\"not_supported\": true, \"resources\": [\"*\"], \"privileges\": [\"my-app:login\" ]}] }";
+        final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> RoleDescriptor.parse("test", new BytesArray(badJson), false, XContentType.JSON));
+        assertThat(ex.getMessage(), containsString("not_supported"));
     }
 
     public void testSerialization() throws Exception {
@@ -125,11 +217,24 @@ public class RoleDescriptorTests extends ESTestCase {
                         .query("{\"query\": {\"match_all\": {}}}")
                         .build()
         };
+        final RoleDescriptor.ApplicationResourcePrivileges[] applicationPrivileges = {
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .application("my_app")
+                .privileges("read", "write")
+                .resources("*")
+                .build()
+        };
+        final ConditionalClusterPrivilege[] conditionalClusterPrivileges = {
+            new ConditionalClusterPrivileges.ManageApplicationPrivileges(new LinkedHashSet<>(Arrays.asList("app01", "app02")))
+        };
+
         Map<String, Object> metadata = randomBoolean() ? MetadataUtils.DEFAULT_RESERVED_METADATA : null;
-        final RoleDescriptor descriptor =
-                new RoleDescriptor("test", new String[] { "all", "none" }, groups, new String[] { "sudo" }, metadata);
+        final RoleDescriptor descriptor = new RoleDescriptor("test", new String[]{"all", "none"}, groups, applicationPrivileges,
+            conditionalClusterPrivileges, new String[] { "sudo" }, metadata, null);
         RoleDescriptor.writeTo(descriptor, output);
-        StreamInput streamInput = ByteBufferStreamInput.wrap(BytesReference.toBytes(output.bytes()));
+        final NamedWriteableRegistry registry = new NamedWriteableRegistry(new XPackClientPlugin(Settings.EMPTY).getNamedWriteables());
+        StreamInput streamInput = new NamedWriteableAwareStreamInput(ByteBufferStreamInput.wrap(BytesReference.toBytes(output.bytes())),
+            registry);
         final RoleDescriptor serialized = RoleDescriptor.readFrom(streamInput);
         assertEquals(descriptor, serialized);
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
index b33e93d0806..0c2ab1ecc76 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
@@ -7,6 +7,10 @@ package org.elasticsearch.xpack.security.authz.store;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.get.GetAction;
+import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -22,18 +26,26 @@ import org.elasticsearch.license.License.OperationMode;
 import org.elasticsearch.license.TestUtils.UpdatableLicenseState;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateAction;
+import org.elasticsearch.xpack.core.security.action.user.PutUserAction;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
@@ -42,6 +54,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.mock.orig.Mockito.times;
 import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
@@ -103,7 +116,8 @@ public class CompositeRolesStoreTests extends ESTestCase {
         when(fileRolesStore.roleDescriptors(Collections.singleton("fls_dls"))).thenReturn(Collections.singleton(flsDlsRole));
         when(fileRolesStore.roleDescriptors(Collections.singleton("no_fls_dls"))).thenReturn(Collections.singleton(noFlsDlsRole));
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(Settings.EMPTY, fileRolesStore, mock(NativeRolesStore.class),
-                mock(ReservedRolesStore.class), Collections.emptyList(), new ThreadContext(Settings.EMPTY), licenseState);
+                mock(ReservedRolesStore.class), mock(NativePrivilegeStore.class), Collections.emptyList(),
+                new ThreadContext(Settings.EMPTY), licenseState);
 
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
@@ -163,7 +177,8 @@ public class CompositeRolesStoreTests extends ESTestCase {
         when(fileRolesStore.roleDescriptors(Collections.singleton("fls_dls"))).thenReturn(Collections.singleton(flsDlsRole));
         when(fileRolesStore.roleDescriptors(Collections.singleton("no_fls_dls"))).thenReturn(Collections.singleton(noFlsDlsRole));
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(Settings.EMPTY, fileRolesStore, mock(NativeRolesStore.class),
-                mock(ReservedRolesStore.class), Collections.emptyList(), new ThreadContext(Settings.EMPTY), licenseState);
+                mock(ReservedRolesStore.class), mock(NativePrivilegeStore.class), Collections.emptyList(),
+                new ThreadContext(Settings.EMPTY), licenseState);
 
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
@@ -196,7 +211,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
 
         final CompositeRolesStore compositeRolesStore =
                 new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
-                        Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
+                        mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
                         new XPackLicenseState(SECURITY_ENABLED_SETTINGS));
         verify(fileRolesStore).addListener(any(Runnable.class)); // adds a listener in ctor
 
@@ -274,8 +289,8 @@ public class CompositeRolesStoreTests extends ESTestCase {
 
         final CompositeRolesStore compositeRolesStore =
                 new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
-                                Arrays.asList(inMemoryProvider1, inMemoryProvider2), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                                new XPackLicenseState(SECURITY_ENABLED_SETTINGS));
+                                mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, inMemoryProvider2),
+                                new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS));
 
         final Set<String> roleNames = Sets.newHashSet("roleA", "roleB", "unknown");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -328,7 +343,9 @@ public class CompositeRolesStoreTests extends ESTestCase {
                         .build()
         }, null);
         FieldPermissionsCache cache = new FieldPermissionsCache(Settings.EMPTY);
-        Role role = CompositeRolesStore.buildRoleFromDescriptors(Sets.newHashSet(flsRole, addsL1Fields), cache);
+        PlainActionFuture<Role> future = new PlainActionFuture<>();
+        CompositeRolesStore.buildRoleFromDescriptors(Sets.newHashSet(flsRole, addsL1Fields), cache, null, future);
+        Role role = future.actionGet();
 
         MetaData metaData = MetaData.builder()
                 .put(new IndexMetaData.Builder("test")
@@ -343,6 +360,111 @@ public class CompositeRolesStoreTests extends ESTestCase {
         assertTrue(acls.get("test").getFieldPermissions().grantsAccessTo("L3.foo"));
     }
 
+    public void testMergingBasicRoles() {
+        final TransportRequest request1 = mock(TransportRequest.class);
+        final TransportRequest request2 = mock(TransportRequest.class);
+        final TransportRequest request3 = mock(TransportRequest.class);
+
+        ConditionalClusterPrivilege ccp1 = mock(ConditionalClusterPrivilege.class);
+        when(ccp1.getPrivilege()).thenReturn(ClusterPrivilege.MANAGE_SECURITY);
+        when(ccp1.getRequestPredicate()).thenReturn(req -> req == request1);
+        RoleDescriptor role1 = new RoleDescriptor("r1", new String[]{"monitor"}, new IndicesPrivileges[]{
+            IndicesPrivileges.builder()
+                .indices("abc-*", "xyz-*")
+                .privileges("read")
+                .build(),
+            IndicesPrivileges.builder()
+                .indices("ind-1-*")
+                .privileges("all")
+                .build(),
+        }, new RoleDescriptor.ApplicationResourcePrivileges[]{
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .application("app1")
+                .resources("user/*")
+                .privileges("read", "write")
+                .build(),
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .application("app1")
+                .resources("settings/*")
+                .privileges("read")
+                .build()
+        }, new ConditionalClusterPrivilege[] { ccp1 },
+        new String[]{"app-user-1"}, null, null);
+
+        ConditionalClusterPrivilege ccp2 = mock(ConditionalClusterPrivilege.class);
+        when(ccp2.getPrivilege()).thenReturn(ClusterPrivilege.MANAGE_SECURITY);
+        when(ccp2.getRequestPredicate()).thenReturn(req -> req == request2);
+        RoleDescriptor role2 = new RoleDescriptor("r2", new String[]{"manage_saml"}, new IndicesPrivileges[]{
+            IndicesPrivileges.builder()
+                .indices("abc-*", "ind-2-*")
+                .privileges("all")
+                .build()
+        }, new RoleDescriptor.ApplicationResourcePrivileges[]{
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .application("app2a")
+                .resources("*")
+                .privileges("all")
+                .build(),
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .application("app2b")
+                .resources("*")
+                .privileges("read")
+                .build()
+        }, new ConditionalClusterPrivilege[] { ccp2 },
+        new String[]{"app-user-2"}, null, null);
+
+        FieldPermissionsCache cache = new FieldPermissionsCache(Settings.EMPTY);
+        PlainActionFuture<Role> future = new PlainActionFuture<>();
+        final NativePrivilegeStore privilegeStore = mock(NativePrivilegeStore.class);
+        doAnswer(inv -> {
+            assertTrue(inv.getArguments().length == 3);
+            ActionListener<Collection<ApplicationPrivilegeDescriptor>> listener
+                = (ActionListener<Collection<ApplicationPrivilegeDescriptor>>) inv.getArguments()[2];
+            Set<ApplicationPrivilegeDescriptor> set = new HashSet<>();
+            Arrays.asList("app1", "app2a", "app2b").forEach(
+                app -> Arrays.asList("read", "write", "all").forEach(
+                    perm -> set.add(
+                        new ApplicationPrivilegeDescriptor(app, perm, Collections.emptySet(), Collections.emptyMap())
+                    )));
+            listener.onResponse(set);
+            return null;
+        }).when(privilegeStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
+        CompositeRolesStore.buildRoleFromDescriptors(Sets.newHashSet(role1, role2), cache, privilegeStore, future);
+        Role role = future.actionGet();
+
+        assertThat(role.cluster().check(ClusterStateAction.NAME, randomFrom(request1, request2, request3)), equalTo(true));
+        assertThat(role.cluster().check(SamlAuthenticateAction.NAME, randomFrom(request1, request2, request3)), equalTo(true));
+        assertThat(role.cluster().check(ClusterUpdateSettingsAction.NAME, randomFrom(request1, request2, request3)), equalTo(false));
+
+        assertThat(role.cluster().check(PutUserAction.NAME, randomFrom(request1, request2)), equalTo(true));
+        assertThat(role.cluster().check(PutUserAction.NAME, request3), equalTo(false));
+
+        final Predicate<String> allowedRead = role.indices().allowedIndicesMatcher(GetAction.NAME);
+        assertThat(allowedRead.test("abc-123"), equalTo(true));
+        assertThat(allowedRead.test("xyz-000"), equalTo(true));
+        assertThat(allowedRead.test("ind-1-a"), equalTo(true));
+        assertThat(allowedRead.test("ind-2-a"), equalTo(true));
+        assertThat(allowedRead.test("foo"), equalTo(false));
+        assertThat(allowedRead.test("abc"), equalTo(false));
+        assertThat(allowedRead.test("xyz"), equalTo(false));
+        assertThat(allowedRead.test("ind-3-a"), equalTo(false));
+
+        final Predicate<String> allowedWrite = role.indices().allowedIndicesMatcher(IndexAction.NAME);
+        assertThat(allowedWrite.test("abc-123"), equalTo(true));
+        assertThat(allowedWrite.test("xyz-000"), equalTo(false));
+        assertThat(allowedWrite.test("ind-1-a"), equalTo(true));
+        assertThat(allowedWrite.test("ind-2-a"), equalTo(true));
+        assertThat(allowedWrite.test("foo"), equalTo(false));
+        assertThat(allowedWrite.test("abc"), equalTo(false));
+        assertThat(allowedWrite.test("xyz"), equalTo(false));
+        assertThat(allowedWrite.test("ind-3-a"), equalTo(false));
+
+        role.application().grants(new ApplicationPrivilege("app1", "app1-read", "write"), "user/joe");
+        role.application().grants(new ApplicationPrivilege("app1", "app1-read", "read"), "settings/hostname");
+        role.application().grants(new ApplicationPrivilege("app2a", "app2a-all", "all"), "user/joe");
+        role.application().grants(new ApplicationPrivilege("app2b", "app2b-read", "read"), "settings/hostname");
+    }
+
     public void testCustomRolesProviderFailures() throws Exception {
         final FileRolesStore fileRolesStore = mock(FileRolesStore.class);
         when(fileRolesStore.roleDescriptors(anySetOf(String.class))).thenReturn(Collections.emptySet());
@@ -370,8 +492,8 @@ public class CompositeRolesStoreTests extends ESTestCase {
 
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
-                                    Arrays.asList(inMemoryProvider1, failingProvider), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                                    new XPackLicenseState(SECURITY_ENABLED_SETTINGS));
+                                    mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, failingProvider),
+                                    new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS));
 
         final Set<String> roleNames = Sets.newHashSet("roleA", "roleB", "unknown");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -411,7 +533,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         // these licenses don't allow custom role providers
         xPackLicenseState.update(randomFrom(OperationMode.BASIC, OperationMode.GOLD, OperationMode.STANDARD), true, null);
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
-            Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
+            Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState);
 
         Set<String> roleNames = Sets.newHashSet("roleA");
@@ -424,7 +546,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         assertEquals(0, role.indices().groups().length);
 
         compositeRolesStore = new CompositeRolesStore(
-            Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
+            Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState);
         // these licenses allow custom role providers
         xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), true, null);
@@ -439,7 +561,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
 
         // license expired, don't allow custom role providers
         compositeRolesStore = new CompositeRolesStore(
-            Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
+            Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState);
         xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), false, null);
         roleNames = Sets.newHashSet("roleA");
@@ -459,7 +581,8 @@ public class CompositeRolesStoreTests extends ESTestCase {
 
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
                 Settings.EMPTY, mock(FileRolesStore.class), mock(NativeRolesStore.class), mock(ReservedRolesStore.class),
-                Collections.emptyList(), new ThreadContext(Settings.EMPTY), new XPackLicenseState(SECURITY_ENABLED_SETTINGS)) {
+                mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(Settings.EMPTY),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS)) {
             @Override
             public void invalidateAll() {
                 numInvalidation.incrementAndGet();
@@ -502,9 +625,10 @@ public class CompositeRolesStoreTests extends ESTestCase {
     public void testCacheClearOnIndexOutOfDateChange() {
         final AtomicInteger numInvalidation = new AtomicInteger(0);
 
-        CompositeRolesStore compositeRolesStore = new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, mock(FileRolesStore.class),
-                mock(NativeRolesStore.class), mock(ReservedRolesStore.class),
-                Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS)) {
+        CompositeRolesStore compositeRolesStore = new CompositeRolesStore(SECURITY_ENABLED_SETTINGS,
+                mock(FileRolesStore.class), mock(NativeRolesStore.class), mock(ReservedRolesStore.class),
+                mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS)) {
             @Override
             public void invalidateAll() {
                 numInvalidation.incrementAndGet();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java
index 14be1e260db..1e2428e7779 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java
@@ -16,6 +16,7 @@ import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.audit.logfile.CapturingLogger;
@@ -123,7 +124,7 @@ public class FileRolesStoreTests extends ESTestCase {
         assertThat(role, notNullValue());
         assertThat(role.names(), equalTo(new String[] { "role3" }));
         assertThat(role.cluster(), notNullValue());
-        assertThat(role.cluster(), is(ClusterPermission.NONE));
+        assertThat(role.cluster(), is(ClusterPermission.SimpleClusterPermission.NONE));
         assertThat(role.indices(), notNullValue());
         assertThat(role.indices().groups(), notNullValue());
         assertThat(role.indices().groups().length, is(1));
@@ -147,7 +148,7 @@ public class FileRolesStoreTests extends ESTestCase {
         assertThat(role, notNullValue());
         assertThat(role.names(), equalTo(new String[] { "role_run_as" }));
         assertThat(role.cluster(), notNullValue());
-        assertThat(role.cluster(), is(ClusterPermission.NONE));
+        assertThat(role.cluster(), is(ClusterPermission.SimpleClusterPermission.NONE));
         assertThat(role.indices(), is(IndicesPermission.NONE));
         assertThat(role.runAs(), notNullValue());
         assertThat(role.runAs().check("user1"), is(true));
@@ -160,7 +161,7 @@ public class FileRolesStoreTests extends ESTestCase {
         assertThat(role, notNullValue());
         assertThat(role.names(), equalTo(new String[] { "role_run_as1" }));
         assertThat(role.cluster(), notNullValue());
-        assertThat(role.cluster(), is(ClusterPermission.NONE));
+        assertThat(role.cluster(), is(ClusterPermission.SimpleClusterPermission.NONE));
         assertThat(role.indices(), is(IndicesPermission.NONE));
         assertThat(role.runAs(), notNullValue());
         assertThat(role.runAs().check("user1"), is(true));
@@ -173,7 +174,7 @@ public class FileRolesStoreTests extends ESTestCase {
         assertThat(role, notNullValue());
         assertThat(role.names(), equalTo(new String[] { "role_fields" }));
         assertThat(role.cluster(), notNullValue());
-        assertThat(role.cluster(), is(ClusterPermission.NONE));
+        assertThat(role.cluster(), is(ClusterPermission.SimpleClusterPermission.NONE));
         assertThat(role.runAs(), is(RunAsPermission.NONE));
         assertThat(role.indices(), notNullValue());
         assertThat(role.indices().groups(), notNullValue());
@@ -195,7 +196,7 @@ public class FileRolesStoreTests extends ESTestCase {
         assertThat(role, notNullValue());
         assertThat(role.names(), equalTo(new String[] { "role_query" }));
         assertThat(role.cluster(), notNullValue());
-        assertThat(role.cluster(), is(ClusterPermission.NONE));
+        assertThat(role.cluster(), is(ClusterPermission.SimpleClusterPermission.NONE));
         assertThat(role.runAs(), is(RunAsPermission.NONE));
         assertThat(role.indices(), notNullValue());
         assertThat(role.indices().groups(), notNullValue());
@@ -216,7 +217,7 @@ public class FileRolesStoreTests extends ESTestCase {
         assertThat(role, notNullValue());
         assertThat(role.names(), equalTo(new String[] { "role_query_fields" }));
         assertThat(role.cluster(), notNullValue());
-        assertThat(role.cluster(), is(ClusterPermission.NONE));
+        assertThat(role.cluster(), is(ClusterPermission.SimpleClusterPermission.NONE));
         assertThat(role.runAs(), is(RunAsPermission.NONE));
         assertThat(role.indices(), notNullValue());
         assertThat(role.indices().groups(), notNullValue());
@@ -341,14 +342,15 @@ public class FileRolesStoreTests extends ESTestCase {
                 fail("Waited too long for the updated file to be picked up");
             }
 
+            final TransportRequest request = mock(TransportRequest.class);
             descriptors = store.roleDescriptors(Collections.singleton("role5"));
             assertThat(descriptors, notNullValue());
             assertEquals(1, descriptors.size());
             Role role = Role.builder(descriptors.iterator().next(), null).build();
             assertThat(role, notNullValue());
             assertThat(role.names(), equalTo(new String[] { "role5" }));
-            assertThat(role.cluster().check("cluster:monitor/foo/bar"), is(true));
-            assertThat(role.cluster().check("cluster:admin/foo/bar"), is(false));
+            assertThat(role.cluster().check("cluster:monitor/foo/bar", request), is(true));
+            assertThat(role.cluster().check("cluster:admin/foo/bar", request), is(false));
 
         } finally {
             if (watcherService != null) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
new file mode 100644
index 00000000000..89058cf4a8b
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
@@ -0,0 +1,300 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authz.store;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchResponseSections;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequest;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.security.support.SecurityIndexManager;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.util.set.Sets.newHashSet;
+import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.not;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@TestLogging("org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore:TRACE")
+public class NativePrivilegeStoreTests extends ESTestCase {
+
+    private NativePrivilegeStore store;
+    private List<ActionRequest> requests;
+    private AtomicReference<ActionListener> listener;
+    private Client client;
+
+    @Before
+    public void setup() {
+        requests = new ArrayList<>();
+        listener = new AtomicReference<>();
+        client = new NoOpClient(getTestName()) {
+            @Override
+            protected <Request extends ActionRequest, Response extends ActionResponse>
+            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+                NativePrivilegeStoreTests.this.requests.add(request);
+                NativePrivilegeStoreTests.this.listener.set(listener);
+            }
+        };
+        final SecurityIndexManager securityIndex = mock(SecurityIndexManager.class);
+        when(securityIndex.isAvailable()).thenReturn(true);
+        Mockito.doAnswer(invocationOnMock -> {
+            assertThat(invocationOnMock.getArguments().length, equalTo(2));
+            assertThat(invocationOnMock.getArguments()[1], instanceOf(Runnable.class));
+            ((Runnable) invocationOnMock.getArguments()[1]).run();
+            return null;
+        }).when(securityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class));
+        store = new NativePrivilegeStore(Settings.EMPTY, client, securityIndex);
+    }
+
+    @After
+    public void cleanup() {
+        client.close();
+    }
+
+    public void testGetSinglePrivilegeByName() throws Exception {
+        final ApplicationPrivilegeDescriptor sourcePrivilege = new ApplicationPrivilegeDescriptor("myapp", "admin",
+            newHashSet("action:admin/*", "action:login", "data:read/*"), emptyMap()
+        );
+
+        final PlainActionFuture<ApplicationPrivilegeDescriptor> future = new PlainActionFuture<>();
+        store.getPrivilege("myapp", "admin", future);
+        assertThat(requests, iterableWithSize(1));
+        assertThat(requests.get(0), instanceOf(GetRequest.class));
+        GetRequest request = (GetRequest) requests.get(0);
+        assertThat(request.index(), equalTo(SecurityIndexManager.SECURITY_INDEX_NAME));
+        assertThat(request.type(), equalTo("doc"));
+        assertThat(request.id(), equalTo("application-privilege_myapp:admin"));
+
+        final String docSource = Strings.toString(sourcePrivilege);
+        listener.get().onResponse(new GetResponse(
+            new GetResult(request.index(), request.type(), request.id(), 1L, true, new BytesArray(docSource), emptyMap())
+        ));
+        final ApplicationPrivilegeDescriptor getPrivilege = future.get(1, TimeUnit.SECONDS);
+        assertThat(getPrivilege, equalTo(sourcePrivilege));
+    }
+
+    public void testGetMissingPrivilege() throws Exception {
+        final PlainActionFuture<ApplicationPrivilegeDescriptor> future = new PlainActionFuture<>();
+        store.getPrivilege("myapp", "admin", future);
+        assertThat(requests, iterableWithSize(1));
+        assertThat(requests.get(0), instanceOf(GetRequest.class));
+        GetRequest request = (GetRequest) requests.get(0);
+        assertThat(request.index(), equalTo(SecurityIndexManager.SECURITY_INDEX_NAME));
+        assertThat(request.type(), equalTo("doc"));
+        assertThat(request.id(), equalTo("application-privilege_myapp:admin"));
+
+        listener.get().onResponse(new GetResponse(
+            new GetResult(request.index(), request.type(), request.id(), -1, false, null, emptyMap())
+        ));
+        final ApplicationPrivilegeDescriptor getPrivilege = future.get(1, TimeUnit.SECONDS);
+        assertThat(getPrivilege, Matchers.nullValue());
+    }
+
+    public void testGetPrivilegesByApplicationName() throws Exception {
+        final List<ApplicationPrivilegeDescriptor> sourcePrivileges = Arrays.asList(
+            new ApplicationPrivilegeDescriptor("myapp", "admin", newHashSet("action:admin/*", "action:login", "data:read/*"), emptyMap()),
+            new ApplicationPrivilegeDescriptor("myapp", "user", newHashSet("action:login", "data:read/*"), emptyMap()),
+            new ApplicationPrivilegeDescriptor("myapp", "author", newHashSet("action:login", "data:read/*", "data:write/*"), emptyMap())
+        );
+
+        final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
+        store.getPrivileges(Arrays.asList("myapp", "yourapp"), null, future);
+        assertThat(requests, iterableWithSize(1));
+        assertThat(requests.get(0), instanceOf(SearchRequest.class));
+        SearchRequest request = (SearchRequest) requests.get(0);
+        assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME));
+
+        final String query = Strings.toString(request.source().query());
+        assertThat(query, containsString("{\"terms\":{\"application\":[\"myapp\",\"yourapp\"]"));
+        assertThat(query, containsString("{\"term\":{\"type\":{\"value\":\"application-privilege\""));
+
+        final SearchHit[] hits = buildHits(sourcePrivileges);
+        listener.get().onResponse(new SearchResponse(new SearchResponseSections(
+            new SearchHits(hits, hits.length, 0f), null, null, false, false, null, 1), "_scrollId1", 1, 1, 0, 1, null, null));
+
+        assertResult(sourcePrivileges, future);
+    }
+
+    public void testGetAllPrivileges() throws Exception {
+        final List<ApplicationPrivilegeDescriptor> sourcePrivileges = Arrays.asList(
+            new ApplicationPrivilegeDescriptor("app1", "admin", newHashSet("action:admin/*", "action:login", "data:read/*"), emptyMap()),
+            new ApplicationPrivilegeDescriptor("app2", "user", newHashSet("action:login", "data:read/*"), emptyMap()),
+            new ApplicationPrivilegeDescriptor("app3", "all", newHashSet("*"), emptyMap())
+        );
+
+        final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
+        store.getPrivileges(null, null, future);
+        assertThat(requests, iterableWithSize(1));
+        assertThat(requests.get(0), instanceOf(SearchRequest.class));
+        SearchRequest request = (SearchRequest) requests.get(0);
+        assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME));
+
+        final String query = Strings.toString(request.source().query());
+        assertThat(query, containsString("{\"term\":{\"type\":{\"value\":\"application-privilege\""));
+        assertThat(query, not(containsString("{\"terms\"")));
+
+        final SearchHit[] hits = buildHits(sourcePrivileges);
+        listener.get().onResponse(new SearchResponse(new SearchResponseSections(
+            new SearchHits(hits, hits.length, 0f), null, null, false, false, null, 1), "_scrollId1", 1, 1, 0, 1, null, null));
+
+        assertResult(sourcePrivileges, future);
+    }
+
+    public void testPutPrivileges() throws Exception {
+        final List<ApplicationPrivilegeDescriptor> putPrivileges = Arrays.asList(
+            new ApplicationPrivilegeDescriptor("app1", "admin", newHashSet("action:admin/*", "action:login", "data:read/*"), emptyMap()),
+            new ApplicationPrivilegeDescriptor("app1", "user", newHashSet("action:login", "data:read/*"), emptyMap()),
+            new ApplicationPrivilegeDescriptor("app2", "all", newHashSet("*"), emptyMap())
+        );
+
+        final PlainActionFuture<Map<String, List<String>>> future = new PlainActionFuture<>();
+        store.putPrivileges(putPrivileges, WriteRequest.RefreshPolicy.IMMEDIATE, future);
+        assertThat(requests, iterableWithSize(putPrivileges.size()));
+        assertThat(requests, everyItem(instanceOf(IndexRequest.class)));
+
+        final List<IndexRequest> indexRequests = new ArrayList<>(requests.size());
+        requests.stream().map(IndexRequest.class::cast).forEach(indexRequests::add);
+        requests.clear();
+
+        final ActionListener indexListener = listener.get();
+        final String uuid = UUIDs.randomBase64UUID(random());
+        for (int i = 0; i < putPrivileges.size(); i++) {
+            ApplicationPrivilegeDescriptor privilege = putPrivileges.get(i);
+            IndexRequest request = indexRequests.get(i);
+            assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME));
+            assertThat(request.type(), equalTo("doc"));
+            assertThat(request.id(), equalTo(
+                "application-privilege_" + privilege.getApplication() + ":" + privilege.getName()
+            ));
+            final XContentBuilder builder = privilege.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), true);
+            assertThat(request.source(), equalTo(BytesReference.bytes(builder)));
+            final boolean created = privilege.getName().equals("user") == false;
+            indexListener.onResponse(new IndexResponse(
+                new ShardId(SecurityIndexManager.SECURITY_INDEX_NAME, uuid, i),
+                request.type(), request.id(), 1, 1, 1, created
+            ));
+        }
+
+        awaitBusy(() -> requests.size() > 0, 1, TimeUnit.SECONDS);
+
+        assertThat(requests, iterableWithSize(1));
+        assertThat(requests.get(0), instanceOf(ClearRolesCacheRequest.class));
+        listener.get().onResponse(null);
+
+        final Map<String, List<String>> map = future.actionGet();
+        assertThat(map.entrySet(), iterableWithSize(2));
+        assertThat(map.get("app1"), iterableWithSize(1));
+        assertThat(map.get("app2"), iterableWithSize(1));
+        assertThat(map.get("app1"), contains("admin"));
+        assertThat(map.get("app2"), contains("all"));
+    }
+
+    public void testDeletePrivileges() throws Exception {
+        final List<String> privilegeNames = Arrays.asList("p1", "p2", "p3");
+
+        final PlainActionFuture<Map<String, List<String>>> future = new PlainActionFuture<>();
+        store.deletePrivileges("app1", privilegeNames, WriteRequest.RefreshPolicy.IMMEDIATE, future);
+        assertThat(requests, iterableWithSize(privilegeNames.size()));
+        assertThat(requests, everyItem(instanceOf(DeleteRequest.class)));
+
+        final List<DeleteRequest> deletes = new ArrayList<>(requests.size());
+        requests.stream().map(DeleteRequest.class::cast).forEach(deletes::add);
+        requests.clear();
+
+        final ActionListener deleteListener = listener.get();
+        final String uuid = UUIDs.randomBase64UUID(random());
+        for (int i = 0; i < privilegeNames.size(); i++) {
+            String name = privilegeNames.get(i);
+            DeleteRequest request = deletes.get(i);
+            assertThat(request.indices(), arrayContaining(SecurityIndexManager.SECURITY_INDEX_NAME));
+            assertThat(request.type(), equalTo("doc"));
+            assertThat(request.id(), equalTo("application-privilege_app1:" + name));
+            final boolean found = name.equals("p2") == false;
+            deleteListener.onResponse(new DeleteResponse(
+                new ShardId(SecurityIndexManager.SECURITY_INDEX_NAME, uuid, i),
+                request.type(), request.id(), 1, 1, 1, found
+            ));
+        }
+
+        awaitBusy(() -> requests.size() > 0, 1, TimeUnit.SECONDS);
+
+        assertThat(requests, iterableWithSize(1));
+        assertThat(requests.get(0), instanceOf(ClearRolesCacheRequest.class));
+        listener.get().onResponse(null);
+
+        final Map<String, List<String>> map = future.actionGet();
+        assertThat(map.entrySet(), iterableWithSize(1));
+        assertThat(map.get("app1"), iterableWithSize(2));
+        assertThat(map.get("app1"), containsInAnyOrder("p1", "p3"));
+    }
+
+    private SearchHit[] buildHits(List<ApplicationPrivilegeDescriptor> sourcePrivileges) {
+        final SearchHit[] hits = new SearchHit[sourcePrivileges.size()];
+        for (int i = 0; i < hits.length; i++) {
+            final ApplicationPrivilegeDescriptor p = sourcePrivileges.get(i);
+            hits[i] = new SearchHit(i, "application-privilege_" + p.getApplication() + ":" + p.getName(), null, null);
+            hits[i].sourceRef(new BytesArray(Strings.toString(p)));
+        }
+        return hits;
+    }
+
+    private void assertResult(List<ApplicationPrivilegeDescriptor> sourcePrivileges,
+                              PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future) throws Exception {
+        final Collection<ApplicationPrivilegeDescriptor> getPrivileges = future.get(1, TimeUnit.SECONDS);
+        assertThat(getPrivileges, iterableWithSize(sourcePrivileges.size()));
+        assertThat(new HashSet<>(getPrivileges), equalTo(new HashSet<>(sourcePrivileges)));
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/HasPrivilegesRestResponseTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/HasPrivilegesRestResponseTests.java
index 601cabf4f84..645abbc8f1a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/HasPrivilegesRestResponseTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/HasPrivilegesRestResponseTests.java
@@ -5,10 +5,6 @@
  */
 package org.elasticsearch.xpack.security.rest.action.user;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -19,6 +15,10 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
 import org.elasticsearch.xpack.security.rest.action.user.RestHasPrivilegesAction.HasPrivilegesRestResponseBuilder;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.mockito.Mockito.mock;
@@ -30,13 +30,13 @@ public class HasPrivilegesRestResponseTests extends ESTestCase {
         final HasPrivilegesResponse actionResponse = new HasPrivilegesResponse(false,
                 Collections.singletonMap("manage", true),
                 Arrays.asList(
-                        new HasPrivilegesResponse.IndexPrivileges("staff",
+                        new HasPrivilegesResponse.ResourcePrivileges("staff",
                                 MapBuilder.<String, Boolean>newMapBuilder(new LinkedHashMap<>())
                                         .put("read", true).put("index", true).put("delete", false).put("manage", false).map()),
-                        new HasPrivilegesResponse.IndexPrivileges("customers",
+                        new HasPrivilegesResponse.ResourcePrivileges("customers",
                                 MapBuilder.<String, Boolean>newMapBuilder(new LinkedHashMap<>())
                                         .put("read", true).put("index", true).put("delete", true).put("manage", false).map())
-                ));
+                ), Collections.emptyMap());
         final XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
         final RestResponse rest = response.buildResponse(actionResponse, builder);
 
@@ -50,6 +50,8 @@ public class HasPrivilegesRestResponseTests extends ESTestCase {
                 "\"index\":{" +
                 "\"staff\":{\"read\":true,\"index\":true,\"delete\":false,\"manage\":false}," +
                 "\"customers\":{\"read\":true,\"index\":true,\"delete\":true,\"manage\":false}" +
-                "}}"));
+                "}," +
+                "\"application\":{}" +
+                "}"));
     }
 }
\ No newline at end of file
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_privileges.json
new file mode 100644
index 00000000000..6086e46eade
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.delete_privileges.json
@@ -0,0 +1,30 @@
+{
+  "xpack.security.delete_privileges": {
+    "documentation": "TODO",
+    "methods": [ "DELETE" ],
+    "url": {
+      "path": "/_xpack/security/privilege/{application}/{name}",
+      "paths": [ "/_xpack/security/privilege/{application}/{name}" ],
+      "parts": {
+        "application": {
+          "type" : "string",
+          "description" : "Application name",
+          "required" : true
+        },
+        "name": {
+          "type" : "string",
+          "description" : "Privilege name",
+          "required" : true
+        }
+      },
+      "params": {
+        "refresh": {
+          "type" : "enum",
+          "options": ["true", "false", "wait_for"],
+          "description" : "If `true` (the default) then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` then do nothing with refreshes."
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_privileges.json
new file mode 100644
index 00000000000..4286ffa954b
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.get_privileges.json
@@ -0,0 +1,24 @@
+{
+  "xpack.security.get_privileges": {
+    "documentation": "TODO",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/_xpack/security/privilege/{application}/{name}",
+      "paths": [ "/_xpack/security/privilege/{application}/{name}" ],
+      "parts": {
+        "application": {
+          "type" : "string",
+          "description" : "Application name",
+          "required" : false
+        },
+        "name": {
+          "type" : "string",
+          "description" : "Privilege name",
+          "required" : false
+        }
+      },
+      "params": {}
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.has_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.has_privileges.json
new file mode 100644
index 00000000000..64b15ae9c02
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.has_privileges.json
@@ -0,0 +1,22 @@
+{
+  "xpack.security.has_privileges": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-privileges.html",
+    "methods": [ "GET", "POST" ],
+    "url": {
+      "path": "/_xpack/security/user/_has_privileges",
+      "paths": [ "/_xpack/security/user/_has_privileges", "/_xpack/security/user/{user}/_has_privileges" ],
+      "parts": {
+        "user": {
+          "type" : "string",
+          "description" : "Username",
+          "required" : false
+        }
+      },
+      "params": {}
+    },
+    "body": {
+      "description" : "The privileges to test",
+      "required" : true
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privilege.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privilege.json
new file mode 100644
index 00000000000..3d453682c64
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privilege.json
@@ -0,0 +1,33 @@
+{
+  "xpack.security.put_privilege": {
+    "documentation": "TODO",
+    "methods": [ "POST", "PUT" ],
+    "url": {
+      "path": "/_xpack/security/privilege/{application}/{name}",
+      "paths": [ "/_xpack/security/privilege/{application}/{name}" ],
+      "parts": {
+        "application": {
+          "type" : "string",
+          "description" : "Application name",
+          "required" : true
+        },
+        "name": {
+          "type" : "string",
+          "description" : "Privilege name",
+          "required" : true
+        }
+      },
+      "params": {
+        "refresh": {
+          "type" : "enum",
+          "options": ["true", "false", "wait_for"],
+          "description" : "If `true` (the default) then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` then do nothing with refreshes."
+        }
+      }
+    },
+    "body": {
+      "description" : "The privilege to add",
+      "required" : true
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privileges.json
new file mode 100644
index 00000000000..07eb5417158
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.security.put_privileges.json
@@ -0,0 +1,27 @@
+{
+  "xpack.security.put_privileges": {
+    "documentation": "TODO",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/_xpack/security/privilege/",
+      "paths": [
+        "/_xpack/security/privilege/"
+      ],
+      "params": {
+        "refresh": {
+          "type": "enum",
+          "options": [
+            "true",
+            "false",
+            "wait_for"
+          ],
+          "description": "If `true` (the default) then refresh the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` then do nothing with refreshes."
+        }
+      }
+    },
+    "body": {
+      "description" : "The privilege(s) to add",
+      "required" : true
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/10_basic.yml
new file mode 100644
index 00000000000..e8dddf21535
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/10_basic.yml
@@ -0,0 +1,324 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+---
+teardown:
+  - do:
+      xpack.security.delete_privileges:
+        application: app
+        name: "p1,p2,p3"
+        ignore: 404
+  - do:
+      xpack.security.delete_privileges:
+        application: app2
+        name: "p1"
+        ignore: 404
+  - do:
+      xpack.security.delete_privileges:
+        application: app3
+        name: "p1,p2,p3,p4"
+        ignore: 404
+  - do:
+      xpack.security.delete_privileges:
+        application: app4
+        name: "p1"
+        ignore: 404
+---
+"Test put and get privileges":
+  # Single privilege, with names in URL
+  - do:
+      xpack.security.put_privilege:
+        application: app
+        name: p1
+        body:  >
+            {
+              "application": "app",
+              "name": "p1",
+              "actions": [ "data:read/*" , "action:login" ],
+              "metadata": {
+                "key1" : "val1a",
+                "key2" : "val2a"
+              }
+            }
+  - match: { "app.p1" : { created: true } }
+
+  # Multiple privileges, no names in URL
+  - do:
+      xpack.security.put_privileges:
+        body: >
+          {
+            "app": {
+              "p2": {
+                "application": "app",
+                "name": "p2",
+                "actions": [ "data:read/*" , "action:login" ],
+                "metadata": {
+                  "key1" : "val1b",
+                  "key2" : "val2b"
+                }
+              },
+              "p3": {
+                "application": "app",
+                "name": "p3",
+                "actions": [ "data:write/*" , "action:login" ],
+                "metadata": {
+                  "key1" : "val1c",
+                  "key2" : "val2c"
+                }
+              }
+            },
+            "app2" : {
+              "p1" : {
+                "application": "app2",
+                "name": "p1",
+                "actions": [ "*" ]
+              }
+            }
+          }
+  - match: { "app.p2" : { created: true } }
+  - match: { "app.p3" : { created: true } }
+  - match: { "app2.p1" : { created: true } }
+
+  # Update existing privilege, with names in URL
+  - do:
+      xpack.security.put_privilege:
+        application: app
+        name: p1
+        body:  >
+            {
+              "application": "app",
+              "name": "p1",
+              "actions": [ "data:read/*" , "action:login" ],
+              "metadata": {
+                "key3" : "val3"
+              }
+            }
+  - match: { "app.p1" : { created: false } }
+
+  # Get the privilege back
+  - do:
+      xpack.security.get_privileges:
+        application: app
+        name: p1
+
+  - match: {
+      "app.p1" : {
+        "application": "app",
+        "name": "p1",
+        "actions": [ "data:read/*" , "action:login" ],
+        "metadata": {
+          "key3" : "val3"
+        }
+      }
+    }
+
+  # Get 2 privileges back
+  - do:
+      xpack.security.get_privileges:
+        application: app
+        name: p1,p2
+
+  - match: {
+      "app.p1" : {
+        "application": "app",
+        "name": "p1",
+        "actions": [ "data:read/*" , "action:login" ],
+        "metadata": {
+          "key3" : "val3"
+        }
+      }
+    }
+  - match: {
+      "app.p2" : {
+        "application": "app",
+        "name": "p2",
+        "actions": [ "data:read/*" , "action:login" ],
+        "metadata": {
+          "key1" : "val1b",
+          "key2" : "val2b"
+        }
+      }
+    }
+
+ # Get all (3) privileges back for "app"
+  - do:
+      xpack.security.get_privileges:
+        application: "app"
+        name: ""
+
+  - match: {
+      "app.p1" : {
+        "application": "app",
+        "name": "p1",
+        "actions": [ "data:read/*" , "action:login" ],
+        "metadata": {
+          "key3" : "val3"
+        }
+      }
+    }
+  - match: {
+      "app.p2" : {
+        "application": "app",
+        "name": "p2",
+        "actions": [ "data:read/*" , "action:login" ],
+        "metadata": {
+          "key1" : "val1b",
+          "key2" : "val2b"
+        }
+      }
+    }
+  - match: {
+      "app.p3" : {
+        "application": "app",
+        "name": "p3",
+        "actions": [ "data:write/*" , "action:login" ],
+        "metadata": {
+          "key1" : "val1c",
+          "key2" : "val2c"
+        }
+      }
+    }
+
+ # Get all (4) privileges back for all apps
+  - do:
+      xpack.security.get_privileges:
+        application: ""
+        name: ""
+
+  - match: {
+      "app.p1" : {
+        "application": "app",
+        "name": "p1",
+        "actions": [ "data:read/*" , "action:login" ],
+        "metadata": {
+          "key3" : "val3"
+        }
+      }
+    }
+  - match: {
+      "app.p2" : {
+        "application": "app",
+        "name": "p2",
+        "actions": [ "data:read/*" , "action:login" ],
+        "metadata": {
+          "key1" : "val1b",
+          "key2" : "val2b"
+        }
+      }
+    }
+  - match: {
+      "app.p3" : {
+        "application": "app",
+        "name": "p3",
+        "actions": [ "data:write/*" , "action:login" ],
+        "metadata": {
+          "key1" : "val1c",
+          "key2" : "val2c"
+        }
+      }
+    }
+  - match: {
+      "app2.p1" : {
+        "application": "app2",
+        "name": "p1",
+        "actions": [ "*" ],
+        "metadata": { }
+      }
+    }
+
+---
+"Test put and delete privileges":
+  # Store some privileges
+  - do:
+      xpack.security.put_privileges:
+        body:  >
+          {
+            "app3": {
+              "p1": {
+                "application": "app3",
+                "name": "p1",
+                "actions": [ "data:read/*" ]
+              },
+              "p2": {
+                "application": "app3",
+                "name": "p2",
+                "actions": [ "data:write/*" ]
+              },
+              "p3": {
+                "application": "app3",
+                "name": "p3",
+                "actions": [ "data:write/*", "data:read/*" ]
+              },
+              "p4": {
+                "application": "app3",
+                "name": "p4",
+                "actions": [ "*" ]
+              }
+            },
+            "app4": {
+              "p1": {
+                "application": "app4",
+                "name": "p1",
+                "actions": [ "*" ]
+              }
+            }
+          }
+  - match: { "app3.p1" : { created: true } }
+  - match: { "app3.p2" : { created: true } }
+  - match: { "app3.p3" : { created: true } }
+  - match: { "app3.p4" : { created: true } }
+  - match: { "app4.p1" : { created: true } }
+
+  # Delete 1 privilege
+  - do:
+      xpack.security.delete_privileges:
+        application: app3
+        name: p1
+
+  - match: { "app3.p1" : { "found" : true } }
+
+  # Delete 2 more privileges (p2, p3)
+  # and try to delete two that don't exist (p1, p0)
+  - do:
+      xpack.security.delete_privileges:
+        application: app3
+        name: p1,p2,p3,p0
+
+  - match: { "app3.p1" : { "found" : false} }
+  - match: { "app3.p2" : { "found" : true } }
+  - match: { "app3.p3" : { "found" : true } }
+  - match: { "app3.p0" : { "found" : false} }
+
+  # Check the deleted privileges are gone
+  - do:
+      catch: missing
+      xpack.security.get_privileges:
+        application: app3
+        name: p1,p2,p3
+
+  # Check the non-deleted privileges are there
+  - do:
+      xpack.security.get_privileges:
+        application: ""
+        name: ""
+  - match: {
+      "app3.p4" : {
+        "application": "app3",
+        "name": "p4",
+        "actions": [ "*" ],
+        "metadata": { }
+      }
+    }
+  - match: {
+      "app4.p1" : {
+        "application": "app4",
+        "name": "p1",
+        "actions": [ "*" ],
+        "metadata": { }
+      }
+    }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/20_has_application_privs.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/20_has_application_privs.yml
new file mode 100644
index 00000000000..1860564863f
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/20_has_application_privs.yml
@@ -0,0 +1,190 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  # Create some privileges
+  - do:
+      xpack.security.put_privileges:
+        body: >
+          {
+            "myapp": {
+              "user": {
+                "application": "myapp",
+                "name": "user",
+                "actions": [ "action:login", "version:1.0.*" ]
+              },
+              "read": {
+                "application": "myapp",
+                "name": "read",
+                "actions": [ "data:read/*" ]
+              },
+              "write": {
+                "application": "myapp",
+                "name": "write",
+                "actions": [ "data:write/*" ]
+              }
+            }
+          }
+
+  # Store 2 test roles
+  - do:
+      xpack.security.put_role:
+        name: "myapp_engineering_read"
+        body:  >
+            {
+              "cluster": [],
+              "indices": [
+                {
+                  "names": "engineering-*",
+                  "privileges": ["read"]
+                }
+              ],
+              "applications": [
+                {
+                  "application": "myapp",
+                  "privileges": ["user"],
+                  "resources": ["*"]
+                },
+                {
+                  "application": "myapp",
+                  "privileges": ["read"],
+                  "resources": ["engineering/*"]
+                }
+              ]
+            }
+
+  - do:
+      xpack.security.put_role:
+        name: "myapp_engineering_write"
+        body:  >
+            {
+              "cluster": [],
+              "indices": [
+                {
+                  "names": "engineering-*",
+                  "privileges": ["read"]
+                }
+              ],
+              "applications": [
+                {
+                  "application": "myapp",
+                  "privileges": ["user"],
+                  "resources": ["*"]
+                },
+                {
+                  "application": "myapp",
+                  "privileges": ["read", "write"],
+                  "resources": ["engineering/*"]
+                }
+              ]
+            }
+
+  # And a user for each role
+  - do:
+        xpack.security.put_user:
+          username: "eng_read"
+          body:  >
+              {
+                "password": "p@ssw0rd",
+                "roles" : [ "myapp_engineering_read" ]
+              }
+  - do:
+        xpack.security.put_user:
+          username: "eng_write"
+          body:  >
+              {
+                "password": "p@ssw0rd",
+                "roles" : [ "myapp_engineering_write" ]
+              }
+
+---
+teardown:
+  - do:
+      xpack.security.delete_privileges:
+        application: myapp
+        name: "user,read,write"
+        ignore: 404
+
+  - do:
+      xpack.security.delete_user:
+        username: "eng_read"
+        ignore: 404
+
+  - do:
+      xpack.security.delete_user:
+        username: "eng_write"
+        ignore: 404
+
+  - do:
+      xpack.security.delete_role:
+        name: "myapp_engineering_read"
+        ignore: 404
+
+  - do:
+      xpack.security.delete_role:
+        name: "myapp_engineering_write"
+        ignore: 404
+---
+"Test has_privileges with application-privileges":
+  - do:
+      headers: { Authorization: "Basic ZW5nX3JlYWQ6cEBzc3cwcmQ=" } # eng_read
+      xpack.security.has_privileges:
+        user: null
+        body: >
+          {
+            "index": [
+              {
+                "names" :[ "engineering-logs", "product-logs" ],
+                "privileges" : [ "read", "index", "write" ]
+              }
+            ],
+            "application": [
+              {
+                "application" : "myapp",
+                "resources" : [ "*" ],
+                "privileges" : [ "action:login", "version:1.0.3" ]
+              },
+              {
+                "application" : "myapp",
+                "resources" : [ "engineering/logs/*", "product/logs/*" ],
+                "privileges" : [ "data:read/log/raw", "data:write/log/raw" ]
+              }
+            ]
+          }
+
+  - match: { "username" : "eng_read" }
+  - match: { "has_all_requested" : false }
+  - match: { "index" : {
+      "engineering-logs" : {
+        "read": true,
+        "index": false,
+        "write": false
+      },
+      "product-logs" : {
+        "read": false,
+        "index": false,
+        "write": false
+      }
+    } }
+  - match: { "application" : {
+      "myapp" : {
+        "*" : {
+          "action:login" : true,
+          "version:1.0.3" : true
+        },
+        "engineering/logs/*" : {
+          "data:read/log/raw" : true,
+          "data:write/log/raw" : false
+        },
+        "product/logs/*" : {
+          "data:read/log/raw" : false,
+          "data:write/log/raw" : false
+        }
+      }
+    } }
+
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/30_superuser.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/30_superuser.yml
new file mode 100644
index 00000000000..cbf08e94d59
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/30_superuser.yml
@@ -0,0 +1,131 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  # Create some privileges
+  - do:
+      xpack.security.put_privileges:
+        body: >
+          {
+            "app01": {
+              "user": {
+                "application": "app01",
+                "name": "user",
+                "actions": [ "action:login" ]
+              },
+              "read": {
+                "application": "app01",
+                "name": "read",
+                "actions": [ "data:read/*" ]
+              },
+              "write": {
+                "application": "app01",
+                "name": "write",
+                "actions": [ "data:write/*" ]
+              }
+            },
+            "app02": {
+              "user": {
+                "application": "app02",
+                "name": "user",
+                "actions": [ "action:login" ]
+              },
+              "read": {
+                "application": "app02",
+                "name": "read",
+                "actions": [ "data:read/*" ]
+              },
+              "write": {
+                "application": "app02",
+                "name": "write",
+                "actions": [ "data:write/*" ]
+              }
+            }
+          }
+
+  # And a superuser
+  - do:
+        xpack.security.put_user:
+          username: "my_admin"
+          body:  >
+              {
+                "password": "admin01",
+                "roles" : [ "superuser" ]
+              }
+  - do:
+        xpack.security.put_user:
+          username: "eng_write"
+          body:  >
+              {
+                "password": "p@ssw0rd",
+                "roles" : [ "myapp_engineering_write" ]
+              }
+
+---
+teardown:
+  - do:
+      xpack.security.delete_privileges:
+        application: app01
+        name: "user,read,write"
+        ignore: 404
+  - do:
+      xpack.security.delete_privileges:
+        application: app02
+        name: "user,read,write"
+        ignore: 404
+
+  - do:
+      xpack.security.delete_user:
+        username: "my_admin"
+        ignore: 404
+
+---
+"Test superuser has all application-privileges":
+  - do:
+      headers: { Authorization: "Basic bXlfYWRtaW46YWRtaW4wMQ==" } # my_admin
+      xpack.security.has_privileges:
+        user: null
+        body: >
+          {
+            "cluster": [ "manage" ],
+            "index": [
+              {
+                "names" :[ "*"  ],
+                "privileges" : [ "read", "index", "write" ]
+              }
+            ],
+            "application": [
+              {
+                "application" : "app01",
+                "resources" : [ "*" ],
+                "privileges" : [ "action:login", "data:read/secrets" ]
+              },
+              {
+                "application" : "app02",
+                "resources" : [ "thing/1" ],
+                "privileges" : [ "data:write/thing" ]
+              }
+            ]
+          }
+
+  - match: { "username" : "my_admin" }
+  - match: { "has_all_requested" : true }
+  - match: { "application" : {
+      "app01" : {
+        "*" : {
+          "action:login" : true,
+          "data:read/secrets" : true
+        }
+      },
+      "app02" : {
+        "thing/1" : {
+          "data:write/thing" : true
+        }
+      }
+    } }
+
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/roles/40_global_privileges.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/roles/40_global_privileges.yml
new file mode 100644
index 00000000000..b89efdfe56c
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/roles/40_global_privileges.yml
@@ -0,0 +1,51 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+  - do:
+      xpack.security.put_user:
+        username: "joe"
+        body:  >
+            {
+              "password": "s3krit",
+              "roles" : [ "with_global" ]
+            }
+
+---
+teardown:
+  - do:
+      xpack.security.delete_user:
+        username: "joe"
+        ignore: 404
+  - do:
+      xpack.security.delete_role:
+        name: "with_global"
+        ignore: 404
+
+
+---
+"Test put role with conditional security privileges":
+  - do:
+      xpack.security.put_role:
+        name: "with_global"
+        body:  >
+            {
+              "global": {
+                "application": {
+                  "manage": {
+                    "applications": [ "app1-*" , "app2-*" ]
+                  }
+                }
+              }
+            }
+  - match: { role: { created: true } }
+
+  - do:
+      xpack.security.get_role:
+        name: "with_global"
+  - match: { with_global.global.application.manage.applications.0:  "app1-*" }
+  - match: { with_global.global.application.manage.applications.1:  "app2-*" }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/40_condtional_cluster_priv.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/40_condtional_cluster_priv.yml
new file mode 100644
index 00000000000..b3a1e220690
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/40_condtional_cluster_priv.yml
@@ -0,0 +1,132 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  - do:
+      xpack.security.put_user:
+        username: "test_user"
+        body:  >
+          {
+            "password" : "x-pack-test-password",
+            "roles" : [ "app_manage" ]
+          }
+
+  - do:
+      xpack.security.put_role:
+        name: "app_manage"
+        body:  >
+            {
+              "global": {
+                "application": {
+                  "manage": {
+                    "applications": [ "app" , "app-*" ]
+                  }
+                }
+              }
+            }
+
+  - do:
+      xpack.security.put_privilege:
+        application: app-allow
+        name: read
+        body:  >
+            {
+              "actions": [ "data:read/*" ]
+            }
+
+  - do:
+      xpack.security.put_privilege:
+        application: app_deny
+        name: read
+        body:  >
+            {
+              "actions": [ "data:read/*" ]
+            }
+
+---
+teardown:
+  - do:
+      xpack.security.delete_user:
+        username: "test_user"
+        ignore: 404
+  - do:
+      xpack.security.delete_role:
+        name: "app_manage"
+        ignore: 404
+
+  - do:
+      xpack.security.delete_privileges:
+        application: app
+        name: read
+        ignore: 404
+
+  - do:
+      xpack.security.delete_privileges:
+        application: app-allow
+        name: read
+        ignore: 404
+
+  - do:
+      xpack.security.delete_privileges:
+        application: app_deny
+        name: read
+        ignore: 404
+
+---
+"Test put application privileges when allowed":
+
+  - do:
+      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
+      xpack.security.put_privilege:
+        application: app
+        name: read
+        body:  >
+            {
+              "actions": [ "data:read/*" ]
+            }
+  - match: { "app.read" : { created: true } }
+
+---
+"Test get application privileges when allowed":
+  - do:
+      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
+      xpack.security.get_privileges:
+        application: app-allow
+        name: read
+
+  - match: {
+      "app-allow.read" : {
+        "application": "app-allow",
+        "name": "read",
+        "actions": [ "data:read/*" ],
+        "metadata": {}
+      }
+    }
+
+---
+"Test put application privileges when not allowed":
+  - do:
+      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
+      xpack.security.put_privilege:
+          application: app_deny
+          name: write
+          body:  >
+              {
+                "actions": [ "data:write/*" ]
+              }
+      catch: forbidden
+
+---
+"Test get application privileges when not allowed":
+  - do:
+      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
+      xpack.security.get_privileges:
+        application: app_deny
+        name: read
+      catch: forbidden
+

From c1cc0cef6136d0c92bcb132f5321c653df8544da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Tue, 24 Jul 2018 19:58:34 +0200
Subject: [PATCH 159/260] Add ERR to ranking evaluation documentation (#32314)

This change adds a section about the Expected Reciprocal Rank metric (ERR) to
the Ranking Evaluation documentation.
---
 docs/reference/search/rank-eval.asciidoc | 50 ++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
index ef715dfca8c..81c464b71d5 100644
--- a/docs/reference/search/rank-eval.asciidoc
+++ b/docs/reference/search/rank-eval.asciidoc
@@ -259,6 +259,56 @@ in the query. Defaults to 10.
 |`normalize` | If set to `true`, this metric will calculate the https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG[Normalized DCG].
 |=======================================================================
 
+[float]
+==== Expected Reciprocal Rank (ERR)
+
+Expected Reciprocal Rank (ERR) is an extension of the classical reciprocal rank for the graded relevance case
+(Olivier Chapelle, Donald Metzler, Ya Zhang, and Pierre Grinspan. 2009. http://olivier.chapelle.cc/pub/err.pdf[Expected reciprocal rank for graded relevance].)
+
+It is based on the assumption of a cascade model of search, in which a user scans through ranked search
+results in order and stops at the first document that satisfies the information need. For this reason, it
+is a good metric for question answering and navigation queries, but less so for survey oriented information 
+needs where the user is interested in finding many relevant documents in the top k results.
+
+The metric models the expectation of the reciprocal of the position at which a user stops reading through
+the result list. This means that relevant document in top ranking positions will contribute much to the
+overall score. However, the same document will contribute much less to the score if it appears in a lower rank,
+even more so if there are some relevant (but maybe less relevant) documents preceding it. 
+In this way, the ERR metric discounts documents which are shown after very relevant documents. This introduces 
+a notion of dependency in the ordering of relevant documents that e.g. Precision or DCG don't account for.
+
+[source,js]
+--------------------------------
+GET /twitter/_rank_eval
+{
+    "requests": [
+    {
+        "id": "JFK query", 
+        "request": { "query": { "match_all": {}}},
+        "ratings": []  
+    }],
+    "metric": {
+       "expected_reciprocal_rank": {
+            "maximum_relevance" : 3,
+            "k" : 20
+       }
+    }
+}
+--------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+The `expected_reciprocal_rank` metric takes the following parameters:
+
+[cols="<,<",options="header",]
+|=======================================================================
+|Parameter |Description
+| `maximum_relevance` | Mandatory parameter. The highest relevance grade used in the user supplied
+relevance judgments.
+|`k` | sets the maximum number of documents retrieved per query. This value will act in place of the usual `size` parameter
+in the query. Defaults to 10.
+|=======================================================================
+
 [float]
 === Response format
 

From 29c802f88e92232a2dc448dc4e91443e6e831148 Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Tue, 24 Jul 2018 11:19:07 -0700
Subject: [PATCH 160/260] Rest HL client: Add put license action (#32214)

In the HL REST client we replace the License object with a string, because of
complexity of this class. It is also not really needed on the client side since
end-users are not interacting with the license besides passing it as a string
to the server.

Relates #29827
---
 .../elasticsearch/client/LicenseClient.java   |  66 ++++++
 .../client/RequestConverters.java             |  13 ++
 .../org/elasticsearch/client/XPackClient.java |  13 ++
 .../LicensingDocumentationIT.java             | 106 +++++++++
 .../high-level/licensing/put-license.asciidoc |  66 ++++++
 .../high-level/supported-apis.asciidoc        |   9 +
 .../elasticsearch/license/LicenseService.java |   2 +
 .../elasticsearch/license/LicensesStatus.java |  34 ---
 .../license/LicensingClient.java              |   1 +
 .../license/PutLicenseAction.java             |   1 +
 .../license/PutLicenseRequestBuilder.java     |   1 +
 .../license/PutLicenseResponse.java           | 119 ----------
 .../license/TransportPutLicenseAction.java    |   1 +
 .../license/LicenseTLSTests.java              |   1 +
 .../license/LicensesAcknowledgementTests.java |   2 +
 .../license/LicensesManagerServiceTests.java  |   3 +-
 .../license/LicensesTransportTests.java       |   4 +-
 .../license/PutLicenseResponseTests.java      | 103 ---------
 .../org/elasticsearch/license/TestUtils.java  |   2 +
 .../protocol/xpack/common/ProtocolUtils.java  |  72 ++++++
 .../xpack/license/LicensesStatus.java         |  68 ++++++
 .../xpack/license/PutLicenseRequest.java      |  54 +++++
 .../xpack/license/PutLicenseResponse.java     | 209 ++++++++++++++++++
 .../xpack/common/ProtocolUtilsTests.java      |  72 ++++++
 .../license/PutLicenseResponseTests.java      | 125 +++++++++++
 25 files changed, 889 insertions(+), 258 deletions(-)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
 create mode 100644 docs/java-rest/high-level/licensing/put-license.asciidoc
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesStatus.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseResponse.java
 delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/license/PutLicenseResponseTests.java
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/common/ProtocolUtils.java
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/LicensesStatus.java
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseRequest.java
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponse.java
 create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/common/ProtocolUtilsTests.java
 create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponseTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java
new file mode 100644
index 00000000000..587578f3b35
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseClient.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptySet;
+
+/**
+ * A wrapper for the {@link RestHighLevelClient} that provides methods for
+ * accessing the Elastic License-related methods
+ * <p>
+ * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html">
+ * X-Pack Licensing APIs on elastic.co</a> for more information.
+ */
+public class LicenseClient {
+
+    private final RestHighLevelClient restHighLevelClient;
+
+    LicenseClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Updates license for the cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public PutLicenseResponse putLicense(PutLicenseRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::putLicense, options,
+            PutLicenseResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously updates license for the cluster cluster.
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void putLicenseAsync(PutLicenseRequest request, RequestOptions options, ActionListener<PutLicenseResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::putLicense, options,
+            PutLicenseResponse::fromXContent, listener, emptySet());
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index b486d9fce52..b9360877dfc 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -109,6 +109,7 @@ import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -1139,6 +1140,18 @@ final class RequestConverters {
         return request;
     }
 
+    static Request putLicense(PutLicenseRequest putLicenseRequest) {
+        Request request = new Request(HttpPut.METHOD_NAME, "/_xpack/license");
+        Params parameters = new Params(request);
+        parameters.withTimeout(putLicenseRequest.timeout());
+        parameters.withMasterTimeout(putLicenseRequest.masterNodeTimeout());
+        if (putLicenseRequest.isAcknowledge()) {
+            parameters.putParam("acknowledge", "true");
+        }
+        request.setJsonEntity(putLicenseRequest.getLicenseDefinition());
+        return request;
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
index 4acaadfdb85..1401376527d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
@@ -42,10 +42,12 @@ public final class XPackClient {
 
     private final RestHighLevelClient restHighLevelClient;
     private final WatcherClient watcherClient;
+    private final LicenseClient licenseClient;
 
     XPackClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
         this.watcherClient = new WatcherClient(restHighLevelClient);
+        this.licenseClient = new LicenseClient(restHighLevelClient);
     }
 
     public WatcherClient watcher() {
@@ -100,4 +102,15 @@ public final class XPackClient {
         restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xpackUsage, options,
             XPackUsageResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * A wrapper for the {@link RestHighLevelClient} that provides methods for
+     * accessing the Elastic Licensing APIs.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html">
+     * X-Pack APIs on elastic.co</a> for more information.
+     */
+    public LicenseClient license() {
+        return licenseClient;
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
new file mode 100644
index 00000000000..d620adb7131
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.documentation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.LatchedActionListener;
+import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.protocol.xpack.license.LicensesStatus;
+import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
+
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWith;
+
+/**
+ * Documentation for Licensing APIs in the high level java client.
+ * Code wrapped in {@code tag} and {@code end} tags is included in the docs.
+ */
+public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
+
+    public void testPutLicense() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        String license = "{\"license\": {\"uid\":\"893361dc-9749-4997-93cb-802e3d7fa4a8\",\"type\":\"gold\"," +
+            "\"issue_date_in_millis\":1411948800000,\"expiry_date_in_millis\":1914278399999,\"max_nodes\":1,\"issued_to\":\"issued_to\"," +
+            "\"issuer\":\"issuer\",\"signature\":\"AAAAAgAAAA3U8+YmnvwC+CWsV/mRAAABmC9ZN0hjZDBGYnVyRXpCOW5Bb3FjZDAxOWpSbTVoMVZwUzRxVk1PSm" +
+            "kxakxZdW5IMlhlTHNoN1N2MXMvRFk4d3JTZEx3R3RRZ0pzU3lobWJKZnQvSEFva0ppTHBkWkprZWZSQi9iNmRQNkw1SlpLN0lDalZCS095MXRGN1lIZlpYcVVTTn" +
+            "FrcTE2dzhJZmZrdFQrN3JQeGwxb0U0MXZ0dDJHSERiZTVLOHNzSDByWnpoZEphZHBEZjUrTVBxRENNSXNsWWJjZllaODdzVmEzUjNiWktNWGM5TUhQV2plaUo4Q1" +
+            "JOUml4MXNuL0pSOEhQaVB2azhmUk9QVzhFeTFoM1Q0RnJXSG53MWk2K055c28zSmRnVkF1b2JSQkFLV2VXUmVHNDZ2R3o2VE1qbVNQS2lxOHN5bUErZlNIWkZSVm" +
+            "ZIWEtaSU9wTTJENDVvT1NCYklacUYyK2FwRW9xa0t6dldMbmMzSGtQc3FWOTgzZ3ZUcXMvQkt2RUZwMFJnZzlvL2d2bDRWUzh6UG5pdENGWFRreXNKNkE9PQAAAQ" +
+            "Be8GfzDm6T537Iuuvjetb3xK5dvg0K5NQapv+rczWcQFxgCuzbF8plkgetP1aAGZP4uRESDQPMlOCsx4d0UqqAm9f7GbBQ3l93P+PogInPFeEH9NvOmaAQovmxVM" +
+            "9SE6DsDqlX4cXSO+bgWpXPTd2LmpoQc1fXd6BZ8GeuyYpVHVKp9hVU0tAYjw6HzYOE7+zuO1oJYOxElqy66AnIfkvHrvni+flym3tE7tDTgsDRaz7W3iBhaqiSnt" +
+            "EqabEkvHdPHQdSR99XGaEvnHO1paK01/35iZF6OXHsF7CCj+558GRXiVxzueOe7TsGSSt8g7YjZwV9bRCyU7oB4B/nidgI\"}}";
+        {
+            //tag::put-license-execute
+            PutLicenseRequest request = new PutLicenseRequest();
+            request.setLicenseDefinition(license);  // <1>
+            request.setAcknowledge(false);          // <2>
+
+            PutLicenseResponse response = client.xpack().license().putLicense(request, RequestOptions.DEFAULT);
+            //end::put-license-execute
+
+            //tag::put-license-response
+            LicensesStatus status = response.status();                  // <1>
+            assertEquals(status, LicensesStatus.VALID);                 // <2>
+            boolean acknowledged = response.isAcknowledged();           // <3>
+            String acknowledgeHeader = response.acknowledgeHeader();    // <4>
+            Map<String, String[]> acknowledgeMessages = response.acknowledgeMessages();  // <5>
+            //end::put-license-response
+
+            assertFalse(acknowledged); // Should fail because we are trying to downgrade from platinum trial to gold
+            assertThat(acknowledgeHeader, startsWith("This license update requires acknowledgement."));
+            assertThat(acknowledgeMessages.keySet(), not(hasSize(0)));
+        }
+        {
+            PutLicenseRequest request = new PutLicenseRequest();
+            // tag::put-license-execute-listener
+            ActionListener<PutLicenseResponse> listener = new ActionListener<PutLicenseResponse>() {
+                @Override
+                public void onResponse(PutLicenseResponse indexResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::put-license-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::put-license-execute-async
+            client.xpack().license().putLicenseAsync(
+                    request, RequestOptions.DEFAULT, listener); // <1>
+            // end::put-license-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+}
diff --git a/docs/java-rest/high-level/licensing/put-license.asciidoc b/docs/java-rest/high-level/licensing/put-license.asciidoc
new file mode 100644
index 00000000000..2486ad8010d
--- /dev/null
+++ b/docs/java-rest/high-level/licensing/put-license.asciidoc
@@ -0,0 +1,66 @@
+[[java-rest-high-put-license]]
+=== Update License
+
+[[java-rest-high-put-license-execution]]
+==== Execution
+
+The license can be added or updated using the `putLicense()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[put-license-execute]
+--------------------------------------------------
+<1> Set the categories of information to retrieve. The the default is to
+return no information which is useful for checking if {xpack} is installed
+but not much else.
+<2> A JSON document containing the license information.
+
+[[java-rest-high-put-license-response]]
+==== Response
+
+The returned `PutLicenseResponse` contains the `LicensesStatus`,
+`acknowledged` flag and possible acknowledge messages. The acknowledge messages
+are present if you previously had a license with more features than one you
+are trying to update and you didn't set the `acknowledge` flag to `true`. In this case
+you need to display the messages to the end user and if they agree, resubmit the
+license with the `acknowledge` flag set to `true`. Please note that the request will
+still return a 200 return code even if requires an acknowledgement. So, it is
+necessary to check the `acknowledged` flag.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[put-license-response]
+--------------------------------------------------
+<1> The status of the license
+<2> Make sure that the license is valid.
+<3> Check the acknowledge flag.
+<4> It should be true if license is acknowledge.
+<5> Otherwise we can see the acknowledge messages in `acknowledgeHeader()`
+<6> and check component-specific messages in `acknowledgeMessages()`.
+
+[[java-rest-high-put-license-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[put-license-execute-async]
+--------------------------------------------------
+<1> The `PutLicenseRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `PutLicenseResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/LicensingDocumentationIT.java[put-license-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index d952870677b..25fbcaaaeaa 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -186,3 +186,12 @@ The Java High Level REST Client supports the following Scripts APIs:
 
 include::script/get_script.asciidoc[]
 include::script/delete_script.asciidoc[]
+
+
+== Licensing APIs
+
+The Java High Level REST Client supports the following Licensing APIs:
+
+* <<java-rest-high-put-license>>
+
+include::licensing/put-license.asciidoc[]
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
index d5e38fc0cb8..505d2c55fa6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
@@ -28,6 +28,8 @@ import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse;
+import org.elasticsearch.protocol.xpack.license.LicensesStatus;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesStatus.java
deleted file mode 100644
index 91e0d7239cf..00000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesStatus.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.license;
-
-public enum LicensesStatus {
-    VALID((byte) 0),
-    INVALID((byte) 1),
-    EXPIRED((byte) 2);
-
-    private final byte id;
-
-    LicensesStatus(byte id) {
-        this.id = id;
-    }
-
-    public int id() {
-        return id;
-    }
-
-    public static LicensesStatus fromId(int id) {
-        if (id == 0) {
-            return VALID;
-        } else if (id == 1) {
-            return INVALID;
-        } else if (id == 2) {
-            return EXPIRED;
-        } else {
-            throw new IllegalStateException("no valid LicensesStatus for id=" + id);
-        }
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
index 07979d15ea5..14a059e9e01 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
@@ -7,6 +7,7 @@ package org.elasticsearch.license;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
 public class LicensingClient {
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseAction.java
index d93957a9d8b..497b203f413 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseAction.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
 public class PutLicenseAction extends Action<PutLicenseResponse> {
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequestBuilder.java
index b7c93d03cd5..bc5201a8f1f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequestBuilder.java
@@ -9,6 +9,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
 /**
  * Register license request builder
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseResponse.java
deleted file mode 100644
index c85bb068da3..00000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseResponse.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-public class PutLicenseResponse extends AcknowledgedResponse {
-
-    private LicensesStatus status;
-    private Map<String, String[]> acknowledgeMessages;
-    private String acknowledgeHeader;
-
-    PutLicenseResponse() {
-    }
-
-    public PutLicenseResponse(boolean acknowledged, LicensesStatus status) {
-        this(acknowledged, status, null, Collections.<String, String[]>emptyMap());
-    }
-
-    public PutLicenseResponse(boolean acknowledged, LicensesStatus status, String acknowledgeHeader,
-                              Map<String, String[]> acknowledgeMessages) {
-        super(acknowledged);
-        this.status = status;
-        this.acknowledgeHeader = acknowledgeHeader;
-        this.acknowledgeMessages = acknowledgeMessages;
-    }
-
-    public LicensesStatus status() {
-        return status;
-    }
-
-    public Map<String, String[]> acknowledgeMessages() {
-        return acknowledgeMessages;
-    }
-
-    public String acknowledgeHeader() {
-        return acknowledgeHeader;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        status = LicensesStatus.fromId(in.readVInt());
-        acknowledgeHeader = in.readOptionalString();
-        int size = in.readVInt();
-        Map<String, String[]> acknowledgeMessages = new HashMap<>(size);
-        for (int i = 0; i < size; i++) {
-            String feature = in.readString();
-            int nMessages = in.readVInt();
-            String[] messages = new String[nMessages];
-            for (int j = 0; j < nMessages; j++) {
-                messages[j] = in.readString();
-            }
-            acknowledgeMessages.put(feature, messages);
-        }
-        this.acknowledgeMessages = acknowledgeMessages;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVInt(status.id());
-        out.writeOptionalString(acknowledgeHeader);
-        out.writeVInt(acknowledgeMessages.size());
-        for (Map.Entry<String, String[]> entry : acknowledgeMessages.entrySet()) {
-            out.writeString(entry.getKey());
-            out.writeVInt(entry.getValue().length);
-            for (String message : entry.getValue()) {
-                out.writeString(message);
-            }
-        }
-    }
-
-    @Override
-    protected void addCustomFields(XContentBuilder builder, Params params) throws IOException {
-        switch (status) {
-            case VALID:
-                builder.field("license_status", "valid");
-                break;
-            case INVALID:
-                builder.field("license_status", "invalid");
-                break;
-            case EXPIRED:
-                builder.field("license_status", "expired");
-                break;
-            default:
-                throw new IllegalArgumentException("unknown status [" + status + "] found");
-        }
-        if (!acknowledgeMessages.isEmpty()) {
-            builder.startObject("acknowledge");
-            builder.field("message", acknowledgeHeader);
-            for (Map.Entry<String, String[]> entry : acknowledgeMessages.entrySet()) {
-                builder.startArray(entry.getKey());
-                for (String message : entry.getValue()) {
-                    builder.value(message);
-                }
-                builder.endArray();
-            }
-            builder.endObject();
-        }
-    }
-
-    @Override
-    public String toString() {
-        return Strings.toString(this, true, true);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
index 032d1eb6e86..571bfc1413c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
@@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTLSTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTLSTests.java
index 588dbabb9db..754b398cd6c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTLSTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTLSTests.java
@@ -12,6 +12,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
 import java.net.InetAddress;
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesAcknowledgementTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesAcknowledgementTests.java
index 8e1c363c814..06afd2f7ca3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesAcknowledgementTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesAcknowledgementTests.java
@@ -9,6 +9,8 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.license.LicensesStatus;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueHours;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
index 540dbd891bd..c397bd79e28 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
@@ -11,6 +11,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -145,4 +146,4 @@ public class LicensesManagerServiceTests extends ESSingleNodeTestCase {
         }
         assertThat("remove license(s) failed", success.get(), equalTo(true));
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
index a48132ef3d7..aa372eb0356 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
@@ -12,6 +12,8 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.license.LicensesStatus;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -230,4 +232,4 @@ public class LicensesTransportTests extends ESSingleNodeTestCase {
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/PutLicenseResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/PutLicenseResponseTests.java
deleted file mode 100644
index d4b7900fa5b..00000000000
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/PutLicenseResponseTests.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.sameInstance;
-
-public class PutLicenseResponseTests extends ESTestCase {
-    @SuppressWarnings("unchecked")
-    public void testSerialization() throws Exception {
-        boolean acknowledged = randomBoolean();
-        LicensesStatus status = randomFrom(LicensesStatus.VALID, LicensesStatus.INVALID, LicensesStatus.EXPIRED);
-        Map<String, String[]> ackMessages = randomAckMessages();
-
-        PutLicenseResponse response = new PutLicenseResponse(acknowledged, status, "", ackMessages);
-
-        XContentBuilder contentBuilder = XContentFactory.jsonBuilder();
-        response.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
-
-        Map<String, Object> map = XContentHelper.convertToMap(BytesReference.bytes(contentBuilder), false,
-                contentBuilder.contentType()).v2();
-        assertThat(map.containsKey("acknowledged"), equalTo(true));
-        boolean actualAcknowledged = (boolean) map.get("acknowledged");
-        assertThat(actualAcknowledged, equalTo(acknowledged));
-
-        assertThat(map.containsKey("license_status"), equalTo(true));
-        String actualStatus = (String) map.get("license_status");
-        assertThat(actualStatus, equalTo(status.name().toLowerCase(Locale.ROOT)));
-
-        assertThat(map.containsKey("acknowledge"), equalTo(true));
-        Map<String, List<String>> actualAckMessages = (Map<String, List<String>>) map.get("acknowledge");
-        assertTrue(actualAckMessages.containsKey("message"));
-        actualAckMessages.remove("message");
-        assertThat(actualAckMessages.keySet(), equalTo(ackMessages.keySet()));
-        for (Map.Entry<String, List<String>> entry : actualAckMessages.entrySet()) {
-            assertArrayEquals(entry.getValue().toArray(), ackMessages.get(entry.getKey()));
-        }
-    }
-
-    public void testStreamSerialization() throws IOException {
-        boolean acknowledged = randomBoolean();
-        LicensesStatus status = randomFrom(LicensesStatus.VALID, LicensesStatus.INVALID, LicensesStatus.EXPIRED);
-        Map<String, String[]> ackMessages = randomAckMessages();
-
-        // write the steam so that we can attempt to read it back
-        BytesStreamOutput output = new BytesStreamOutput();
-
-        PutLicenseResponse response = new PutLicenseResponse(acknowledged, status, "", ackMessages);
-        // write it out
-        response.writeTo(output);
-
-        StreamInput input = output.bytes().streamInput();
-
-        // read it back in
-        response.readFrom(input);
-
-        assertThat(response.isAcknowledged(), equalTo(acknowledged));
-        assertThat(response.status(), equalTo(status));
-        assertThat(response.acknowledgeMessages(), not(sameInstance(ackMessages)));
-        assertThat(response.acknowledgeMessages().size(), equalTo(ackMessages.size()));
-
-        for (String key : ackMessages.keySet()) {
-            assertArrayEquals(ackMessages.get(key), response.acknowledgeMessages().get(key));
-        }
-    }
-
-    private static Map<String, String[]> randomAckMessages() {
-        int nFeatures = randomIntBetween(1, 5);
-
-        Map<String, String[]> ackMessages = new HashMap<>();
-
-        for (int i = 0; i < nFeatures; i++) {
-            String feature = randomAlphaOfLengthBetween(9, 15);
-            int nMessages = randomIntBetween(1, 5);
-            String[] messages = new String[nMessages];
-            for (int j = 0; j < nMessages; j++) {
-                messages[j] = randomAlphaOfLengthBetween(10, 30);
-            }
-            ackMessages.put(feature, messages);
-        }
-
-        return ackMessages;
-    }
-}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
index 3b7906ae567..afa1a8d6796 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
@@ -20,6 +20,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.license.licensor.LicenseSigner;
+import org.elasticsearch.protocol.xpack.license.LicensesStatus;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 import org.hamcrest.MatcherAssert;
 import org.joda.time.format.DateTimeFormatter;
 import org.junit.Assert;
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/common/ProtocolUtils.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/common/ProtocolUtils.java
new file mode 100644
index 00000000000..e135cdc50e9
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/common/ProtocolUtils.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.protocol.xpack.common;
+
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * Common utilities used for XPack protocol classes
+ */
+public final class ProtocolUtils {
+
+    /**
+     * Implements equals for a map of string arrays
+     *
+     * The map of string arrays is used in some XPack protocol classes but does't work with equal.
+     */
+    public static boolean equals(Map<String, String[]> a, Map<String, String[]> b) {
+        if (a == null) {
+            return b == null;
+        }
+        if (b == null) {
+            return false;
+        }
+        if (a.size() != b.size()) {
+            return false;
+        }
+        for (Map.Entry<String, String[]> entry : a.entrySet()) {
+            String[] val = entry.getValue();
+            String key = entry.getKey();
+            if (val == null) {
+                if (b.get(key) != null || b.containsKey(key) == false) {
+                    return false;
+                }
+            } else {
+                if (Arrays.equals(val, b.get(key)) == false) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Implements hashCode for map of string arrays
+     *
+     * The map of string arrays does't work with hashCode.
+     */
+    public static int hashCode(Map<String, String[]> a) {
+        int hash = 0;
+        for (Map.Entry<String, String[]> entry : a.entrySet())
+            hash += Arrays.hashCode(entry.getValue());
+        return hash;
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/LicensesStatus.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/LicensesStatus.java
new file mode 100644
index 00000000000..2a3ed924fbe
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/LicensesStatus.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.license;
+
+import java.util.Locale;
+
+public enum LicensesStatus {
+    VALID((byte) 0),
+    INVALID((byte) 1),
+    EXPIRED((byte) 2);
+
+    private final byte id;
+
+    LicensesStatus(byte id) {
+        this.id = id;
+    }
+
+    public int id() {
+        return id;
+    }
+
+    public static LicensesStatus fromId(int id) {
+        if (id == 0) {
+            return VALID;
+        } else if (id == 1) {
+            return INVALID;
+        } else if (id == 2) {
+            return EXPIRED;
+        } else {
+            throw new IllegalStateException("no valid LicensesStatus for id=" + id);
+        }
+    }
+
+
+    @Override
+    public String toString() {
+        return this.name().toLowerCase(Locale.ROOT);
+    }
+
+    public static LicensesStatus fromString(String value) {
+        switch (value) {
+            case "valid":
+                return VALID;
+            case "invalid":
+                return INVALID;
+            case "expired":
+                return EXPIRED;
+            default:
+                throw new IllegalArgumentException("unknown licenses status [" + value + "]");
+        }
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseRequest.java
new file mode 100644
index 00000000000..97101a3ccd4
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseRequest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.protocol.xpack.license;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+
+public class PutLicenseRequest extends AcknowledgedRequest<PutLicenseRequest> {
+
+    private String licenseDefinition;
+    private boolean acknowledge = false;
+
+    public PutLicenseRequest() {
+
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    public void setLicenseDefinition(String licenseDefinition) {
+        this.licenseDefinition = licenseDefinition;
+    }
+
+    public String getLicenseDefinition() {
+        return licenseDefinition;
+    }
+
+    public void setAcknowledge(boolean acknowledge) {
+        this.acknowledge = acknowledge;
+    }
+
+    public boolean isAcknowledge() {
+        return acknowledge;
+    }
+}
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponse.java
new file mode 100644
index 00000000000..9c4ff51d92a
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponse.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.protocol.xpack.license;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParseException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.xpack.common.ProtocolUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class PutLicenseResponse extends AcknowledgedResponse {
+
+    private static final ConstructingObjectParser<PutLicenseResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "put_license_response", true, (a, v) -> {
+        boolean acknowledged = (Boolean) a[0];
+        LicensesStatus licensesStatus = LicensesStatus.fromString((String) a[1]);
+        @SuppressWarnings("unchecked") Tuple<String, Map<String, String[]>> acknowledgements = (Tuple<String, Map<String, String[]>>) a[2];
+        if (acknowledgements == null) {
+            return new PutLicenseResponse(acknowledged, licensesStatus);
+        } else {
+            return new PutLicenseResponse(acknowledged, licensesStatus, acknowledgements.v1(), acknowledgements.v2());
+        }
+
+    });
+
+    static {
+        PARSER.declareBoolean(constructorArg(), new ParseField("acknowledged"));
+        PARSER.declareString(constructorArg(), new ParseField("license_status"));
+        PARSER.declareObject(optionalConstructorArg(), (parser, v) -> {
+                Map<String, String[]> acknowledgeMessages = new HashMap<>();
+                String message = null;
+                XContentParser.Token token;
+                String currentFieldName = null;
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        currentFieldName = parser.currentName();
+                    } else {
+                        if (currentFieldName == null) {
+                            throw new XContentParseException(parser.getTokenLocation(), "expected message header or acknowledgement");
+                        }
+                        if ("message".equals(currentFieldName)) {
+                            if (token != XContentParser.Token.VALUE_STRING) {
+                                throw new XContentParseException(parser.getTokenLocation(), "unexpected message header type");
+                            }
+                            message = parser.text();
+                        } else {
+                            if (token != XContentParser.Token.START_ARRAY) {
+                                throw new XContentParseException(parser.getTokenLocation(), "unexpected acknowledgement type");
+                            }
+                            List<String> acknowledgeMessagesList = new ArrayList<>();
+                            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                                if (token != XContentParser.Token.VALUE_STRING) {
+                                    throw new XContentParseException(parser.getTokenLocation(), "unexpected acknowledgement text");
+                                }
+                                acknowledgeMessagesList.add(parser.text());
+                            }
+                            acknowledgeMessages.put(currentFieldName, acknowledgeMessagesList.toArray(new String[0]));
+                        }
+                    }
+                }
+                return new Tuple<>(message, acknowledgeMessages);
+            },
+            new ParseField("acknowledge"));
+    }
+
+    private LicensesStatus status;
+    private Map<String, String[]> acknowledgeMessages;
+    private String acknowledgeHeader;
+
+    public PutLicenseResponse() {
+    }
+
+    public PutLicenseResponse(boolean acknowledged, LicensesStatus status) {
+        this(acknowledged, status, null, Collections.<String, String[]>emptyMap());
+    }
+
+    public PutLicenseResponse(boolean acknowledged, LicensesStatus status, String acknowledgeHeader,
+                              Map<String, String[]> acknowledgeMessages) {
+        super(acknowledged);
+        this.status = status;
+        this.acknowledgeHeader = acknowledgeHeader;
+        this.acknowledgeMessages = acknowledgeMessages;
+    }
+
+    public LicensesStatus status() {
+        return status;
+    }
+
+    public Map<String, String[]> acknowledgeMessages() {
+        return acknowledgeMessages;
+    }
+
+    public String acknowledgeHeader() {
+        return acknowledgeHeader;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        status = LicensesStatus.fromId(in.readVInt());
+        acknowledgeHeader = in.readOptionalString();
+        int size = in.readVInt();
+        Map<String, String[]> acknowledgeMessages = new HashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            String feature = in.readString();
+            int nMessages = in.readVInt();
+            String[] messages = new String[nMessages];
+            for (int j = 0; j < nMessages; j++) {
+                messages[j] = in.readString();
+            }
+            acknowledgeMessages.put(feature, messages);
+        }
+        this.acknowledgeMessages = acknowledgeMessages;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVInt(status.id());
+        out.writeOptionalString(acknowledgeHeader);
+        out.writeVInt(acknowledgeMessages.size());
+        for (Map.Entry<String, String[]> entry : acknowledgeMessages.entrySet()) {
+            out.writeString(entry.getKey());
+            out.writeVInt(entry.getValue().length);
+            for (String message : entry.getValue()) {
+                out.writeString(message);
+            }
+        }
+    }
+
+    @Override
+    protected void addCustomFields(XContentBuilder builder, Params params) throws IOException {
+        builder.field("license_status", status.toString());
+        if (!acknowledgeMessages.isEmpty()) {
+            builder.startObject("acknowledge");
+            builder.field("message", acknowledgeHeader);
+            for (Map.Entry<String, String[]> entry : acknowledgeMessages.entrySet()) {
+                builder.startArray(entry.getKey());
+                for (String message : entry.getValue()) {
+                    builder.value(message);
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+    public static PutLicenseResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        PutLicenseResponse that = (PutLicenseResponse) o;
+
+        return status == that.status &&
+            ProtocolUtils.equals(acknowledgeMessages, that.acknowledgeMessages) &&
+            Objects.equals(acknowledgeHeader, that.acknowledgeHeader);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), status, ProtocolUtils.hashCode(acknowledgeMessages), acknowledgeHeader);
+    }
+
+
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/common/ProtocolUtilsTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/common/ProtocolUtilsTests.java
new file mode 100644
index 00000000000..21470832738
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/common/ProtocolUtilsTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.protocol.xpack.common;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ProtocolUtilsTests  extends ESTestCase {
+
+    public void testMapStringEqualsAndHash() {
+        assertTrue(ProtocolUtils.equals(null, null));
+        assertFalse(ProtocolUtils.equals(null, new HashMap<>()));
+        assertFalse(ProtocolUtils.equals(new HashMap<>(), null));
+
+        Map<String, String[]> a = new HashMap<>();
+        a.put("foo", new String[] { "a", "b" });
+        a.put("bar", new String[] { "b", "c" });
+
+        Map<String, String[]> b = new HashMap<>();
+        b.put("foo", new String[] { "a", "b" });
+
+        assertFalse(ProtocolUtils.equals(a, b));
+        assertFalse(ProtocolUtils.equals(b, a));
+
+        b.put("bar", new String[] { "c", "b" });
+
+        assertFalse(ProtocolUtils.equals(a, b));
+        assertFalse(ProtocolUtils.equals(b, a));
+
+        b.put("bar", new String[] { "b", "c" });
+
+        assertTrue(ProtocolUtils.equals(a, b));
+        assertTrue(ProtocolUtils.equals(b, a));
+        assertEquals(ProtocolUtils.hashCode(a), ProtocolUtils.hashCode(b));
+
+        b.put("baz", new String[] { "b", "c" });
+
+        assertFalse(ProtocolUtils.equals(a, b));
+        assertFalse(ProtocolUtils.equals(b, a));
+
+        a.put("non", null);
+
+        assertFalse(ProtocolUtils.equals(a, b));
+        assertFalse(ProtocolUtils.equals(b, a));
+
+        b.put("non", null);
+        b.remove("baz");
+
+        assertTrue(ProtocolUtils.equals(a, b));
+        assertTrue(ProtocolUtils.equals(b, a));
+        assertEquals(ProtocolUtils.hashCode(a), ProtocolUtils.hashCode(b));
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponseTests.java
new file mode 100644
index 00000000000..0980a07b68a
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponseTests.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.license;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+public class PutLicenseResponseTests extends AbstractStreamableXContentTestCase<PutLicenseResponse> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // The structure of the response is such that unknown fields inside acknowledge cannot be supported since they
+        // are treated as messages from new services
+        return p -> p.startsWith("acknowledge");
+    }
+
+    @Override
+    protected PutLicenseResponse createTestInstance() {
+        boolean acknowledged = randomBoolean();
+        LicensesStatus status = randomFrom(LicensesStatus.VALID, LicensesStatus.INVALID, LicensesStatus.EXPIRED);
+        String messageHeader;
+        Map<String, String[]> ackMessages;
+        if (randomBoolean()) {
+            messageHeader = randomAlphaOfLength(10);
+            ackMessages = randomAckMessages();
+        } else {
+            messageHeader = null;
+            ackMessages = Collections.emptyMap();
+        }
+
+        return new PutLicenseResponse(acknowledged, status, messageHeader, ackMessages);
+    }
+
+    private static Map<String, String[]> randomAckMessages() {
+        int nFeatures = randomIntBetween(1, 5);
+
+        Map<String, String[]> ackMessages = new HashMap<>();
+
+        for (int i = 0; i < nFeatures; i++) {
+            String feature = randomAlphaOfLengthBetween(9, 15);
+            int nMessages = randomIntBetween(1, 5);
+            String[] messages = new String[nMessages];
+            for (int j = 0; j < nMessages; j++) {
+                messages[j] = randomAlphaOfLengthBetween(10, 30);
+            }
+            ackMessages.put(feature, messages);
+        }
+
+        return ackMessages;
+    }
+
+    @Override
+    protected PutLicenseResponse doParseInstance(XContentParser parser) throws IOException {
+        return PutLicenseResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected PutLicenseResponse createBlankInstance() {
+        return new PutLicenseResponse();
+    }
+
+    @Override
+    protected PutLicenseResponse mutateInstance(PutLicenseResponse response) {
+        @SuppressWarnings("unchecked")
+        Function<PutLicenseResponse, PutLicenseResponse> mutator = randomFrom(
+            r -> new PutLicenseResponse(
+                r.isAcknowledged() == false,
+                r.status(),
+                r.acknowledgeHeader(),
+                r.acknowledgeMessages()),
+            r -> new PutLicenseResponse(
+                r.isAcknowledged(),
+                mutateStatus(r.status()),
+                r.acknowledgeHeader(),
+                r.acknowledgeMessages()),
+            r -> {
+                if (r.acknowledgeMessages().isEmpty()) {
+                    return new PutLicenseResponse(
+                        r.isAcknowledged(),
+                        r.status(),
+                        randomAlphaOfLength(10),
+                        randomAckMessages()
+                    );
+                } else {
+                    return new PutLicenseResponse(r.isAcknowledged(), r.status());
+                }
+            }
+
+        );
+        return mutator.apply(response);
+    }
+
+    private LicensesStatus mutateStatus(LicensesStatus status) {
+        return randomValueOtherThan(status, () -> randomFrom(LicensesStatus.values()));
+    }
+
+}

From 32eb4d41b655a8e24ade22a3ee99a3e176ed847e Mon Sep 17 00:00:00 2001
From: debadair <debadair@elastic.co>
Date: Tue, 24 Jul 2018 11:51:52 -0700
Subject: [PATCH 161/260] [DOCS] Removed extraneous callout number.

---
 docs/java-rest/high-level/licensing/put-license.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/java-rest/high-level/licensing/put-license.asciidoc b/docs/java-rest/high-level/licensing/put-license.asciidoc
index 2486ad8010d..7456fdd3c02 100644
--- a/docs/java-rest/high-level/licensing/put-license.asciidoc
+++ b/docs/java-rest/high-level/licensing/put-license.asciidoc
@@ -35,8 +35,8 @@ include-tagged::{doc-tests}/LicensingDocumentationIT.java[put-license-response]
 <2> Make sure that the license is valid.
 <3> Check the acknowledge flag.
 <4> It should be true if license is acknowledge.
-<5> Otherwise we can see the acknowledge messages in `acknowledgeHeader()`
-<6> and check component-specific messages in `acknowledgeMessages()`.
+<5> Otherwise we can see the acknowledge messages in `acknowledgeHeader()` and check
+component-specific messages in `acknowledgeMessages()`.
 
 [[java-rest-high-put-license-async]]
 ==== Asynchronous Execution

From e0b7e4b1c5f13af43daebf37657206d4da380b92 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Tue, 24 Jul 2018 10:54:21 -0600
Subject: [PATCH 162/260] Add V_6_3_3 version constant

Relates to #32329
---
 server/src/main/java/org/elasticsearch/Version.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index f22408ba2e5..da0e3a3b3b1 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -174,6 +174,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_6_3_1 = new Version(V_6_3_1_ID, org.apache.lucene.util.Version.LUCENE_7_3_1);
     public static final int V_6_3_2_ID = 6030299;
     public static final Version V_6_3_2 = new Version(V_6_3_2_ID, org.apache.lucene.util.Version.LUCENE_7_3_1);
+    public static final int V_6_3_3_ID = 6030399;
+    public static final Version V_6_3_3 = new Version(V_6_3_3_ID, org.apache.lucene.util.Version.LUCENE_7_3_1);
     public static final int V_6_4_0_ID = 6040099;
     public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
@@ -196,6 +198,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
                 return V_7_0_0_alpha1;
             case V_6_4_0_ID:
                 return V_6_4_0;
+            case V_6_3_3_ID:
+                return V_6_3_3;
             case V_6_3_2_ID:
                 return V_6_3_2;
             case V_6_3_1_ID:

From be40a6982a9a90879fe198260df6a4cb3c0e8582 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ikakavas@protonmail.com>
Date: Tue, 24 Jul 2018 22:05:00 +0300
Subject: [PATCH 163/260] Introduce fips_mode setting and associated checks
 (#32326)

* Introduce fips_mode setting and associated checks

Introduce xpack.security.fips_mode.enabled setting ( default false)
When it is set to true, a number of Bootstrap checks are performed:
- Check that Secure Settings are of the latest version (3)
- Check that no JKS keystores are configured
- Check that compliant algorithms ( PBKDF2 family ) are used for
  password hashing
---
 .../common/settings/KeyStoreWrapper.java      |  13 ++-
 .../FIPS140JKSKeystoreBootstrapCheck.java     |  54 ++++++++++
 ...asswordHashingAlgorithmBootstrapCheck.java |  43 ++++++++
 .../FIPS140SecureSettingsBootstrapCheck.java  |  52 +++++++++
 .../xpack/security/Security.java              |   6 ++
 ...FIPS140JKSKeystoreBootstrapCheckTests.java |  66 ++++++++++++
 ...rdHashingAlgorithmBootstrapCheckTests.java |  34 ++++++
 ...S140SecureSettingsBootstrapCheckTests.java | 102 ++++++++++++++++++
 8 files changed, 368 insertions(+), 2 deletions(-)
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheckTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheckTests.java

diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
index 3a8a06949b2..eee45743ee3 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -166,6 +166,13 @@ public class KeyStoreWrapper implements SecureSettings {
         this.dataBytes = dataBytes;
     }
 
+    /**
+     * Get the metadata format version for the keystore
+     **/
+    public int getFormatVersion() {
+        return formatVersion;
+    }
+
     /** Returns a path representing the ES keystore in the given config dir. */
     public static Path keystorePath(Path configDir) {
         return configDir.resolve(KEYSTORE_FILENAME);
@@ -593,8 +600,10 @@ public class KeyStoreWrapper implements SecureSettings {
     @Override
     public synchronized void close() {
         this.closed = true;
-        for (Entry entry : entries.get().values()) {
-            Arrays.fill(entry.bytes, (byte)0);
+        if (null != entries.get() && entries.get().isEmpty() == false) {
+            for (Entry entry : entries.get().values()) {
+                Arrays.fill(entry.bytes, (byte) 0);
+            }
         }
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
new file mode 100644
index 00000000000..4a2c7b97195
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.common.settings.Settings;
+
+
+public class FIPS140JKSKeystoreBootstrapCheck implements BootstrapCheck {
+
+    private final boolean fipsModeEnabled;
+
+    FIPS140JKSKeystoreBootstrapCheck(Settings settings) {
+        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+    }
+
+    /**
+     * Test if the node fails the check.
+     *
+     * @param context the bootstrap context
+     * @return the result of the bootstrap check
+     */
+    @Override
+    public BootstrapCheckResult check(BootstrapContext context) {
+
+        if (fipsModeEnabled) {
+            final Settings settings = context.settings;
+            Settings keystoreTypeSettings = settings.filter(k -> k.endsWith("keystore.type"))
+                .filter(k -> settings.get(k).equalsIgnoreCase("jks"));
+            if (keystoreTypeSettings.isEmpty() == false) {
+                return BootstrapCheckResult.failure("JKS Keystores cannot be used in a FIPS 140 compliant JVM. Please " +
+                    "revisit [" + keystoreTypeSettings.toDelimitedString(',') + "] settings");
+            }
+            // Default Keystore type is JKS if not explicitly set
+            Settings keystorePathSettings = settings.filter(k -> k.endsWith("keystore.path"))
+                .filter(k -> settings.hasValue(k.replace(".path", ".type")) == false);
+            if (keystorePathSettings.isEmpty() == false) {
+                return BootstrapCheckResult.failure("JKS Keystores cannot be used in a FIPS 140 compliant JVM. Please " +
+                    "revisit [" + keystorePathSettings.toDelimitedString(',') + "] settings");
+            }
+
+        }
+        return BootstrapCheckResult.success();
+    }
+
+    @Override
+    public boolean alwaysEnforce() {
+        return fipsModeEnabled;
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
new file mode 100644
index 00000000000..7f6d799cf5a
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+import java.util.Locale;
+
+public class FIPS140PasswordHashingAlgorithmBootstrapCheck implements BootstrapCheck {
+
+    private final boolean fipsModeEnabled;
+
+    FIPS140PasswordHashingAlgorithmBootstrapCheck(Settings settings) {
+        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+    }
+
+    /**
+     * Test if the node fails the check.
+     *
+     * @param context the bootstrap context
+     * @return the result of the bootstrap check
+     */
+    @Override
+    public BootstrapCheckResult check(BootstrapContext context) {
+        final String selectedAlgorithm = XPackSettings.PASSWORD_HASHING_ALGORITHM.get(context.settings);
+        if (selectedAlgorithm.toLowerCase(Locale.ROOT).startsWith("pbkdf2") == false) {
+            return BootstrapCheckResult.failure("Only PBKDF2 is allowed for password hashing in a FIPS-140 JVM. Please set the " +
+                "appropriate value for [ " + XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey() + " ] setting.");
+        }
+        return BootstrapCheckResult.success();
+    }
+
+    @Override
+    public boolean alwaysEnforce() {
+        return fipsModeEnabled;
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
new file mode 100644
index 00000000000..c766dd0ffaa
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+public class FIPS140SecureSettingsBootstrapCheck implements BootstrapCheck {
+
+    private final boolean fipsModeEnabled;
+    private final Environment environment;
+
+    FIPS140SecureSettingsBootstrapCheck(Settings settings, Environment environment) {
+        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+        this.environment = environment;
+    }
+
+    /**
+     * Test if the node fails the check.
+     *
+     * @param context the bootstrap context
+     * @return the result of the bootstrap check
+     */
+    @Override
+    public BootstrapCheckResult check(BootstrapContext context) {
+        if (fipsModeEnabled) {
+            try (KeyStoreWrapper secureSettings = KeyStoreWrapper.load(environment.configFile())) {
+                if (secureSettings != null && secureSettings.getFormatVersion() < 3) {
+                    return BootstrapCheckResult.failure("Secure settings store is not of the latest version. Please use " +
+                        "bin/elasticsearch-keystore create to generate a new secure settings store and migrate the secure settings there.");
+                }
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+        return BootstrapCheckResult.success();
+    }
+
+    @Override
+    public boolean alwaysEnforce() {
+        return fipsModeEnabled;
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 92784159700..b9cdd22ba47 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -256,6 +256,8 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         DiscoveryPlugin, MapperPlugin, ExtensiblePlugin {
 
     private static final Logger logger = Loggers.getLogger(Security.class);
+    static final Setting<Boolean> FIPS_MODE_ENABLED =
+        Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope);
 
     static final Setting<List<String>> AUDIT_OUTPUTS_SETTING =
         Setting.listSetting(SecurityField.setting("audit.outputs"),
@@ -305,6 +307,9 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 new PkiRealmBootstrapCheck(getSslService()),
                 new TLSLicenseBootstrapCheck(),
                 new PasswordHashingAlgorithmBootstrapCheck(),
+                new FIPS140SecureSettingsBootstrapCheck(settings, env),
+                new FIPS140JKSKeystoreBootstrapCheck(settings),
+                new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings),
                 new KerberosRealmBootstrapCheck(env)));
             checks.addAll(InternalRealms.getBootstrapChecks(settings, env));
             this.bootstrapChecks = Collections.unmodifiableList(checks);
@@ -592,6 +597,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         }
 
         // The following just apply in node mode
+        settingsList.add(FIPS_MODE_ENABLED);
 
         // IP Filter settings
         IPFilter.addSettings(settingsList);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheckTests.java
new file mode 100644
index 00000000000..1d4da71e11b
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheckTests.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+public class FIPS140JKSKeystoreBootstrapCheckTests extends ESTestCase {
+
+    public void testNoKeystoreIsAllowed() {
+        final Settings.Builder settings = Settings.builder()
+            .put("xpack.security.fips_mode.enabled", "true");
+        assertFalse(new FIPS140JKSKeystoreBootstrapCheck(settings.build()).check(new BootstrapContext(settings.build(), null)).isFailure());
+    }
+
+    public void testSSLKeystoreTypeIsNotAllowed() {
+        final Settings.Builder settings = Settings.builder()
+            .put("xpack.security.fips_mode.enabled", "true")
+            .put("xpack.ssl.keystore.path", "/this/is/the/path")
+            .put("xpack.ssl.keystore.type", "JKS");
+        assertTrue(new FIPS140JKSKeystoreBootstrapCheck(settings.build()).check(new BootstrapContext(settings.build(), null)).isFailure());
+    }
+
+    public void testSSLImplicitKeystoreTypeIsNotAllowed() {
+        final Settings.Builder settings = Settings.builder()
+            .put("xpack.security.fips_mode.enabled", "true")
+            .put("xpack.ssl.keystore.path", "/this/is/the/path")
+            .put("xpack.ssl.keystore.type", "JKS");
+        assertTrue(new FIPS140JKSKeystoreBootstrapCheck(settings.build()).check(new BootstrapContext(settings.build(), null)).isFailure());
+    }
+
+    public void testTransportSSLKeystoreTypeIsNotAllowed() {
+        final Settings.Builder settings = Settings.builder()
+            .put("xpack.security.fips_mode.enabled", "true")
+            .put("xpack.security.transport.ssl.keystore.path", "/this/is/the/path")
+            .put("xpack.security.transport.ssl.keystore.type", "JKS");
+        assertTrue(new FIPS140JKSKeystoreBootstrapCheck(settings.build()).check(new BootstrapContext(settings.build(), null)).isFailure());
+    }
+
+    public void testHttpSSLKeystoreTypeIsNotAllowed() {
+        final Settings.Builder settings = Settings.builder()
+            .put("xpack.security.fips_mode.enabled", "true")
+            .put("xpack.security.http.ssl.keystore.path", "/this/is/the/path")
+            .put("xpack.security.http.ssl.keystore.type", "JKS");
+        assertTrue(new FIPS140JKSKeystoreBootstrapCheck(settings.build()).check(new BootstrapContext(settings.build(), null)).isFailure());
+    }
+
+    public void testRealmKeystoreTypeIsNotAllowed() {
+        final Settings.Builder settings = Settings.builder()
+            .put("xpack.security.fips_mode.enabled", "true")
+            .put("xpack.security.authc.realms.ldap.ssl.keystore.path", "/this/is/the/path")
+            .put("xpack.security.authc.realms.ldap.ssl.keystore.type", "JKS");
+        assertTrue(new FIPS140JKSKeystoreBootstrapCheck(settings.build()).check(new BootstrapContext(settings.build(), null)).isFailure());
+    }
+
+    public void testImplicitRealmKeystoreTypeIsNotAllowed() {
+        final Settings.Builder settings = Settings.builder()
+            .put("xpack.security.fips_mode.enabled", "true")
+            .put("xpack.security.authc.realms.ldap.ssl.keystore.path", "/this/is/the/path");
+        assertTrue(new FIPS140JKSKeystoreBootstrapCheck(settings.build()).check(new BootstrapContext(settings.build(), null)).isFailure());
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
new file mode 100644
index 00000000000..310a6e241e0
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+public class FIPS140PasswordHashingAlgorithmBootstrapCheckTests extends ESTestCase {
+
+    public void testPBKDF2AlgorithmIsAllowed() {
+        Settings settings = Settings.builder().put("xpack.security.fips_mode.enabled", "true").build();
+
+        settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2_10000").build();
+        assertFalse(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
+
+        settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2").build();
+        assertFalse(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
+    }
+
+    public void testBCRYPTAlgorithmIsNotAllowed() {
+        Settings settings = Settings.builder().put("xpack.security.fips_mode.enabled", "true").build();
+        assertTrue(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
+        settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "BCRYPT").build();
+        assertTrue(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
+
+        settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "BCRYPT11").build();
+        assertTrue(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheckTests.java
new file mode 100644
index 00000000000..fb9e7155242
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheckTests.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.test.ESTestCase;
+
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import java.io.ByteArrayOutputStream;
+import java.nio.file.Path;
+import java.security.AccessControlException;
+import java.security.KeyStore;
+import java.util.Base64;
+
+public class FIPS140SecureSettingsBootstrapCheckTests extends ESTestCase {
+
+    public void testLegacySecureSettingsIsNotAllowed() throws Exception {
+        assumeFalse("Can't run in a FIPS JVM, PBE is not available", inFipsJvm());
+        final Settings.Builder builder = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.security.fips_mode.enabled", "true");
+        Environment env = TestEnvironment.newEnvironment(builder.build());
+        generateV2Keystore(env);
+        assertTrue(new FIPS140SecureSettingsBootstrapCheck(builder.build(), env).check(new BootstrapContext(builder.build(),
+            null)).isFailure());
+    }
+
+    public void testCorrectSecureSettingsVersionIsAllowed() throws Exception {
+        final Settings.Builder builder = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.security.fips_mode.enabled", "true");
+        Environment env = TestEnvironment.newEnvironment(builder.build());
+        final KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create();
+        try {
+            keyStoreWrapper.save(env.configFile(), "password".toCharArray());
+        } catch (final AccessControlException e) {
+            if (e.getPermission() instanceof RuntimePermission && e.getPermission().getName().equals("accessUserInformation")) {
+                // this is expected:but we don't care in tests
+            } else {
+                throw e;
+            }
+        }
+        assertFalse(new FIPS140SecureSettingsBootstrapCheck(builder.build(), env).check(new BootstrapContext(builder.build(),
+            null)).isFailure());
+    }
+
+    private void generateV2Keystore(Environment env) throws Exception {
+        Path configDir = env.configFile();
+        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
+        byte[] fileBytes = new byte[20];
+        random().nextBytes(fileBytes);
+        try (IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) {
+
+            CodecUtil.writeHeader(output, "elasticsearch.keystore", 2);
+            output.writeByte((byte) 0); // hasPassword = false
+            output.writeString("PKCS12");
+            output.writeString("PBE"); // string algo
+            output.writeString("PBE"); // file algo
+
+            output.writeVInt(2); // num settings
+            output.writeString("string_setting");
+            output.writeString("STRING");
+            output.writeString("file_setting");
+            output.writeString("FILE");
+
+            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
+            KeyStore keystore = KeyStore.getInstance("PKCS12");
+            keystore.load(null, null);
+            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
+            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
+            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
+
+            byte[] base64Bytes = Base64.getEncoder().encode(fileBytes);
+            char[] chars = new char[base64Bytes.length];
+            for (int i = 0; i < chars.length; ++i) {
+                chars[i] = (char) base64Bytes[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
+            }
+            secretKey = secretFactory.generateSecret(new PBEKeySpec(chars));
+            keystore.setEntry("file_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
+
+            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
+            keystore.store(keystoreBytesStream, new char[0]);
+            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
+            output.writeInt(keystoreBytes.length);
+            output.writeBytes(keystoreBytes, keystoreBytes.length);
+            CodecUtil.writeFooter(output);
+        }
+    }
+}

From 49d4b26f166ff6657cc799d7d77b44a7d6cf9a5d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 24 Jul 2018 12:05:50 -0700
Subject: [PATCH 164/260] Ingest: Support integer and long hex values in
 convert (#32213)

This commit adds checks for hex formatted strings in the convert
processor, allowing strings like `0x1` to be parsed as integer `1`.

closes #32182
---
 .../ingest/common/ConvertProcessor.java       | 12 ++++-
 .../ingest/common/ConvertProcessorTests.java  | 54 +++++++++++++++++++
 2 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
index 264df6f4c5f..2e881b82b59 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
@@ -42,7 +42,11 @@ public final class ConvertProcessor extends AbstractProcessor {
             @Override
             public Object convert(Object value) {
                 try {
-                    return Integer.parseInt(value.toString());
+                    String strValue = value.toString();
+                    if (strValue.startsWith("0x") || strValue.startsWith("-0x")) {
+                        return Integer.decode(strValue);
+                    }
+                    return Integer.parseInt(strValue);
                 } catch(NumberFormatException e) {
                     throw new IllegalArgumentException("unable to convert [" + value + "] to integer", e);
                 }
@@ -52,7 +56,11 @@ public final class ConvertProcessor extends AbstractProcessor {
             @Override
             public Object convert(Object value) {
                 try {
-                    return Long.parseLong(value.toString());
+                    String strValue = value.toString();
+                    if (strValue.startsWith("0x") || strValue.startsWith("-0x")) {
+                        return Long.decode(strValue);
+                    }
+                    return Long.parseLong(strValue);
                 } catch(NumberFormatException e) {
                     throw new IllegalArgumentException("unable to convert [" + value + "] to long", e);
                 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
index 292a03d7d90..f0fc31dab35 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
@@ -49,6 +49,33 @@ public class ConvertProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
     }
 
+    public void testConvertIntHex() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int randomInt = randomInt();
+        String intString = randomInt < 0 ? "-0x" + Integer.toHexString(-randomInt) : "0x" + Integer.toHexString(randomInt);
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, intString);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
+    }
+
+    public void testConvertIntLeadingZero() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "010");
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(10));
+    }
+
+    public void testConvertIntHexError() {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String value = "0x" + randomAlphaOfLengthBetween(1, 10);
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to integer"));
+    }
+
     public void testConvertIntList() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         int numItems = randomIntBetween(1, 10);
@@ -92,6 +119,33 @@ public class ConvertProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong));
     }
 
+    public void testConvertLongHex() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        long randomLong = randomLong();
+        String longString = randomLong < 0 ? "-0x" + Long.toHexString(-randomLong) : "0x" + Long.toHexString(randomLong);
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, longString);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong));
+    }
+
+    public void testConvertLongLeadingZero() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "010");
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L));
+    }
+
+    public void testConvertLongHexError() {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String value = "0x" + randomAlphaOfLengthBetween(1, 10);
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value);
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to long"));
+    }
+
     public void testConvertLongList() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         int numItems = randomIntBetween(1, 10);

From 73be9840dd086dc431d8edfdbfba3ea5b39711c2 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Tue, 24 Jul 2018 13:10:23 -0600
Subject: [PATCH 165/260] [TEST] Fix assumeFalse -> assumeTrue in
 SSLReloadIntegTests

---
 .../java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
index 0090c58ce0c..0835ecee9c2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLReloadIntegTests.java
@@ -93,7 +93,7 @@ public class SSLReloadIntegTests extends SecurityIntegTestCase {
     }
 
     public void testThatSSLConfigurationReloadsOnModification() throws Exception {
-        assumeFalse("test fails on JDK 11 currently", JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0);
+        assumeTrue("test fails on JDK 11 currently", JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0);
         Path keyPath = createTempDir().resolve("testnode_updated.pem");
         Path certPath = createTempDir().resolve("testnode_updated.crt");
         Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem"), keyPath);

From 1690451a9fb3a049b39736b39ea02dbf535b1991 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 24 Jul 2018 13:08:05 -0700
Subject: [PATCH 166/260] Painless: Update More Methods to New Naming Scheme
 (#32305)

This finishes the updating the methods in the PainlessLookupBuilder to the new naming scheme. Mechanical change. Methods include the ones used for copying members in the inheritance hierarchy, calculating shortcuts, and setting the functional interface.
---
 .../painless/lookup/PainlessLookup.java       |  20 +-
 .../lookup/PainlessLookupBuilder.java         | 362 ++++++------------
 2 files changed, 137 insertions(+), 245 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
index 752c0c205dd..bcecd7bbdc7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
@@ -30,30 +30,30 @@ import java.util.Map;
 public final class PainlessLookup {
 
     public Collection<PainlessClass> getStructs() {
-        return javaClassesToPainlessStructs.values();
+        return classesToPainlessClasses.values();
     }
 
-    private final Map<String, Class<?>> painlessTypesToJavaClasses;
-    private final Map<Class<?>, PainlessClass> javaClassesToPainlessStructs;
+    private final Map<String, Class<?>> canonicalClassNamesToClasses;
+    private final Map<Class<?>, PainlessClass> classesToPainlessClasses;
 
-    PainlessLookup(Map<String, Class<?>> painlessTypesToJavaClasses, Map<Class<?>, PainlessClass> javaClassesToPainlessStructs) {
-        this.painlessTypesToJavaClasses = Collections.unmodifiableMap(painlessTypesToJavaClasses);
-        this.javaClassesToPainlessStructs = Collections.unmodifiableMap(javaClassesToPainlessStructs);
+    PainlessLookup(Map<String, Class<?>> canonicalClassNamesToClasses, Map<Class<?>, PainlessClass> classesToPainlessClasses) {
+        this.canonicalClassNamesToClasses = Collections.unmodifiableMap(canonicalClassNamesToClasses);
+        this.classesToPainlessClasses = Collections.unmodifiableMap(classesToPainlessClasses);
     }
 
     public Class<?> getClassFromBinaryName(String painlessType) {
-        return painlessTypesToJavaClasses.get(painlessType.replace('$', '.'));
+        return canonicalClassNamesToClasses.get(painlessType.replace('$', '.'));
     }
 
     public boolean isSimplePainlessType(String painlessType) {
-        return painlessTypesToJavaClasses.containsKey(painlessType);
+        return canonicalClassNamesToClasses.containsKey(painlessType);
     }
 
     public PainlessClass getPainlessStructFromJavaClass(Class<?> clazz) {
-        return javaClassesToPainlessStructs.get(clazz);
+        return classesToPainlessClasses.get(clazz);
     }
 
     public Class<?> getJavaClassFromPainlessType(String painlessType) {
-        return PainlessLookupUtility.canonicalTypeNameToType(painlessType, painlessTypesToJavaClasses);
+        return PainlessLookupUtility.canonicalTypeNameToType(painlessType, canonicalClassNamesToClasses);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 8945c956c27..b15f1f13f20 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -37,7 +37,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Stack;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.CONSTRUCTOR_NAME;
@@ -666,179 +665,6 @@ public class PainlessLookupBuilder {
         }
     }
 
-    private void copyStruct(String struct, List<String> children) {
-        final PainlessClassBuilder owner = classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(struct));
-
-        if (owner == null) {
-            throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
-        }
-
-        for (int count = 0; count < children.size(); ++count) {
-            final PainlessClassBuilder child =
-                    classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(children.get(count)));
-
-            if (child == null) {
-                throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
-                    " not defined for copy to owner struct [" + owner.name + "].");
-            }
-
-            if (!child.clazz.isAssignableFrom(owner.clazz)) {
-                throw new ClassCastException("Child struct [" + child.name + "]" +
-                    " is not a super type of owner struct [" + owner.name + "] in copy.");
-            }
-
-            for (Map.Entry<String,PainlessMethod> kvPair : child.methods.entrySet()) {
-                String methodKey = kvPair.getKey();
-                PainlessMethod method = kvPair.getValue();
-                if (owner.methods.get(methodKey) == null) {
-                    // TODO: some of these are no longer valid or outright don't work
-                    // TODO: since classes may not come from the Painless classloader
-                    // TODO: and it was dependent on the order of the extends which
-                    // TODO: which no longer exists since this is generated automatically
-                    // sanity check, look for missing covariant/generic override
-                    /*if (owner.clazz.isInterface() && child.clazz == Object.class) {
-                        // ok
-                    } else if (child.clazz == Spliterator.OfPrimitive.class || child.clazz == PrimitiveIterator.class) {
-                        // ok, we rely on generics erasure for these (its guaranteed in the javadocs though!!!!)
-                    } else if (Constants.JRE_IS_MINIMUM_JAVA9 && owner.clazz == LocalDate.class) {
-                        // ok, java 9 added covariant override for LocalDate.getEra() to return IsoEra:
-                        // https://bugs.openjdk.java.net/browse/JDK-8072746
-                    } else {
-                        try {
-                            // TODO: we *have* to remove all these public members and use getter methods to encapsulate!
-                            final Class<?> impl;
-                            final Class<?> arguments[];
-                            if (method.augmentation != null) {
-                                impl = method.augmentation;
-                                arguments = new Class<?>[method.arguments.size() + 1];
-                                arguments[0] = method.owner.clazz;
-                                for (int i = 0; i < method.arguments.size(); i++) {
-                                    arguments[i + 1] = method.arguments.get(i).clazz;
-                                }
-                            } else {
-                                impl = owner.clazz;
-                                arguments = new Class<?>[method.arguments.size()];
-                                for (int i = 0; i < method.arguments.size(); i++) {
-                                    arguments[i] = method.arguments.get(i).clazz;
-                                }
-                            }
-                            java.lang.reflect.Method m = impl.getMethod(method.method.getName(), arguments);
-                            if (m.getReturnType() != method.rtn.clazz) {
-                                throw new IllegalStateException("missing covariant override for: " + m + " in " + owner.name);
-                            }
-                            if (m.isBridge() && !Modifier.isVolatile(method.modifiers)) {
-                                // its a bridge in the destination, but not in the source, but it might still be ok, check generics:
-                                java.lang.reflect.Method source = child.clazz.getMethod(method.method.getName(), arguments);
-                                if (!Arrays.equals(source.getGenericParameterTypes(), source.getParameterTypes())) {
-                                    throw new IllegalStateException("missing generic override for: " + m + " in " + owner.name);
-                                }
-                            }
-                        } catch (ReflectiveOperationException e) {
-                            throw new AssertionError(e);
-                        }
-                    }*/
-                    owner.methods.put(methodKey, method);
-                }
-            }
-
-            for (PainlessField field : child.members.values()) {
-                if (owner.members.get(field.name) == null) {
-                    owner.members.put(field.name, new PainlessField(
-                            field.name, field.javaName, owner.clazz, field.clazz, field.modifiers, field.getter, field.setter));
-                }
-            }
-        }
-    }
-
-    /**
-     * Precomputes a more efficient structure for dynamic method/field access.
-     */
-    private void addRuntimeClass(final PainlessClassBuilder struct) {
-        // add all getters/setters
-        for (Map.Entry<String, PainlessMethod> method : struct.methods.entrySet()) {
-            String name = method.getValue().name;
-            PainlessMethod m = method.getValue();
-
-            if (m.arguments.size() == 0 &&
-                name.startsWith("get") &&
-                name.length() > 3 &&
-                Character.isUpperCase(name.charAt(3))) {
-                StringBuilder newName = new StringBuilder();
-                newName.append(Character.toLowerCase(name.charAt(3)));
-                newName.append(name.substring(4));
-                struct.getters.putIfAbsent(newName.toString(), m.handle);
-            } else if (m.arguments.size() == 0 &&
-                name.startsWith("is") &&
-                name.length() > 2 &&
-                Character.isUpperCase(name.charAt(2))) {
-                StringBuilder newName = new StringBuilder();
-                newName.append(Character.toLowerCase(name.charAt(2)));
-                newName.append(name.substring(3));
-                struct.getters.putIfAbsent(newName.toString(), m.handle);
-            }
-
-            if (m.arguments.size() == 1 &&
-                name.startsWith("set") &&
-                name.length() > 3 &&
-                Character.isUpperCase(name.charAt(3))) {
-                StringBuilder newName = new StringBuilder();
-                newName.append(Character.toLowerCase(name.charAt(3)));
-                newName.append(name.substring(4));
-                struct.setters.putIfAbsent(newName.toString(), m.handle);
-            }
-        }
-
-        // add all members
-        for (Map.Entry<String, PainlessField> member : struct.members.entrySet()) {
-            struct.getters.put(member.getKey(), member.getValue().getter);
-            struct.setters.put(member.getKey(), member.getValue().setter);
-        }
-    }
-
-    /** computes the functional interface method for a class, or returns null */
-    private PainlessMethod computeFunctionalInterfaceMethod(PainlessClassBuilder clazz) {
-        if (!clazz.clazz.isInterface()) {
-            return null;
-        }
-        // if its marked with this annotation, we fail if the conditions don't hold (means whitelist bug)
-        // otherwise, this annotation is pretty useless.
-        boolean hasAnnotation = clazz.clazz.isAnnotationPresent(FunctionalInterface.class);
-        List<java.lang.reflect.Method> methods = new ArrayList<>();
-        for (java.lang.reflect.Method m : clazz.clazz.getMethods()) {
-            // default interface methods don't count
-            if (m.isDefault()) {
-                continue;
-            }
-            // static methods don't count
-            if (Modifier.isStatic(m.getModifiers())) {
-                continue;
-            }
-            // if its from Object, it doesn't count
-            try {
-                Object.class.getMethod(m.getName(), m.getParameterTypes());
-                continue;
-            } catch (ReflectiveOperationException e) {
-                // it counts
-            }
-            methods.add(m);
-        }
-        if (methods.size() != 1) {
-            if (hasAnnotation) {
-                throw new IllegalArgumentException("Class: " + clazz.name +
-                    " is marked with FunctionalInterface but doesn't fit the bill: " + methods);
-            }
-            return null;
-        }
-        // inspect the one method found from the reflection API, it should match the whitelist!
-        java.lang.reflect.Method oneMethod = methods.get(0);
-        PainlessMethod painless = clazz.methods.get(buildPainlessMethodKey(oneMethod.getName(), oneMethod.getParameterCount()));
-        if (painless == null || painless.method.equals(org.objectweb.asm.commons.Method.getMethod(oneMethod)) == false) {
-            throw new IllegalArgumentException("Class: " + clazz.name + " is functional but the functional " +
-                "method is not whitelisted!");
-        }
-        return painless;
-    }
-
     public PainlessLookup build() {
         String origin = "internal error";
 
@@ -849,11 +675,11 @@ public class PainlessLookupBuilder {
                 for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
                     String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
                     PainlessClassBuilder painlessStruct =
-                        classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(painlessTypeName));
+                            classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(painlessTypeName));
 
                     if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
                         throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
-                            "[" + painlessStruct.clazz.getName() + "] and [" + whitelistStruct.javaClassName + "]");
+                                "[" + painlessStruct.clazz.getName() + "] and [" + whitelistStruct.javaClassName + "]");
                     }
 
                     origin = whitelistStruct.origin;
@@ -894,78 +720,144 @@ public class PainlessLookupBuilder {
             throw new IllegalArgumentException("error loading whitelist(s) " + origin, exception);
         }
 
-        // goes through each Painless struct and determines the inheritance list,
-        // and then adds all inherited types to the Painless struct's whitelist
+        copyPainlessClassMembers();
+        cacheRuntimeHandles();
+        setFunctionalInterfaceMethods();
+
+        Map<Class<?>, PainlessClass> classesToPainlessClasses = new HashMap<>(classesToPainlessClassBuilders.size());
+
+        for (Map.Entry<Class<?>, PainlessClassBuilder> painlessClassBuilderEntry : classesToPainlessClassBuilders.entrySet()) {
+            classesToPainlessClasses.put(painlessClassBuilderEntry.getKey(), painlessClassBuilderEntry.getValue().build());
+        }
+
+        return new PainlessLookup(canonicalClassNamesToClasses, classesToPainlessClasses);
+    }
+
+    private void copyPainlessClassMembers() {
+        for (Class<?> parentClass : classesToPainlessClassBuilders.keySet()) {
+            copyPainlessInterfaceMembers(parentClass, parentClass);
+
+            Class<?> childClass = parentClass.getSuperclass();
+
+            while (childClass != null) {
+                if (classesToPainlessClassBuilders.containsKey(childClass)) {
+                    copyPainlessClassMembers(childClass, parentClass);
+                }
+
+                copyPainlessInterfaceMembers(childClass, parentClass);
+                childClass = childClass.getSuperclass();
+            }
+        }
+
         for (Class<?> javaClass : classesToPainlessClassBuilders.keySet()) {
-            PainlessClassBuilder painlessStruct = classesToPainlessClassBuilders.get(javaClass);
+            if (javaClass.isInterface()) {
+                copyPainlessClassMembers(Object.class, javaClass);
+            }
+        }
+    }
 
-            List<String> painlessSuperStructs = new ArrayList<>();
-            Class<?> javaSuperClass = painlessStruct.clazz.getSuperclass();
-
-            Stack<Class<?>> javaInteraceLookups = new Stack<>();
-            javaInteraceLookups.push(painlessStruct.clazz);
-
-            // adds super classes to the inheritance list
-            if (javaSuperClass != null && javaSuperClass.isInterface() == false) {
-                while (javaSuperClass != null) {
-                    PainlessClassBuilder painlessSuperStruct = classesToPainlessClassBuilders.get(javaSuperClass);
-
-                    if (painlessSuperStruct != null) {
-                        painlessSuperStructs.add(painlessSuperStruct.name);
-                    }
-
-                    javaInteraceLookups.push(javaSuperClass);
-                    javaSuperClass = javaSuperClass.getSuperclass();
-                }
+    private void copyPainlessInterfaceMembers(Class<?> parentClass, Class<?> targetClass) {
+        for (Class<?> childClass : parentClass.getInterfaces()) {
+            if (classesToPainlessClassBuilders.containsKey(childClass)) {
+                copyPainlessClassMembers(childClass, targetClass);
             }
 
-            // adds all super interfaces to the inheritance list
-            while (javaInteraceLookups.isEmpty() == false) {
-                Class<?> javaInterfaceLookup = javaInteraceLookups.pop();
+            copyPainlessInterfaceMembers(childClass, targetClass);
+        }
+    }
 
-                for (Class<?> javaSuperInterface : javaInterfaceLookup.getInterfaces()) {
-                    PainlessClassBuilder painlessInterfaceStruct = classesToPainlessClassBuilders.get(javaSuperInterface);
+    private void copyPainlessClassMembers(Class<?> originalClass, Class<?> targetClass) {
+        PainlessClassBuilder originalPainlessClassBuilder = classesToPainlessClassBuilders.get(originalClass);
+        PainlessClassBuilder targetPainlessClassBuilder = classesToPainlessClassBuilders.get(targetClass);
 
-                    if (painlessInterfaceStruct != null) {
-                        String painlessInterfaceStructName = painlessInterfaceStruct.name;
+        Objects.requireNonNull(originalPainlessClassBuilder);
+        Objects.requireNonNull(targetPainlessClassBuilder);
 
-                        if (painlessSuperStructs.contains(painlessInterfaceStructName) == false) {
-                            painlessSuperStructs.add(painlessInterfaceStructName);
-                        }
+        for (Map.Entry<String, PainlessMethod> painlessMethodEntry : originalPainlessClassBuilder.methods.entrySet()) {
+            String painlessMethodKey = painlessMethodEntry.getKey();
+            PainlessMethod newPainlessMethod = painlessMethodEntry.getValue();
+            PainlessMethod existingPainlessMethod = targetPainlessClassBuilder.methods.get(painlessMethodKey);
 
-                        for (Class<?> javaPushInterface : javaInterfaceLookup.getInterfaces()) {
-                            javaInteraceLookups.push(javaPushInterface);
-                        }
+            if (existingPainlessMethod == null || existingPainlessMethod.target != newPainlessMethod.target &&
+                    existingPainlessMethod.target.isAssignableFrom(newPainlessMethod.target)) {
+                targetPainlessClassBuilder.methods.put(painlessMethodKey, newPainlessMethod);
+            }
+        }
+
+        for (Map.Entry<String, PainlessField> painlessFieldEntry : originalPainlessClassBuilder.members.entrySet()) {
+            String painlessFieldKey = painlessFieldEntry.getKey();
+            PainlessField newPainlessField = painlessFieldEntry.getValue();
+            PainlessField existingPainlessField = targetPainlessClassBuilder.members.get(painlessFieldKey);
+
+            if (existingPainlessField == null || existingPainlessField.target != newPainlessField.target &&
+                    existingPainlessField.target.isAssignableFrom(newPainlessField.target)) {
+                targetPainlessClassBuilder.members.put(painlessFieldKey, newPainlessField);
+            }
+        }
+    }
+
+    private void cacheRuntimeHandles() {
+        for (PainlessClassBuilder painlessClassBuilder : classesToPainlessClassBuilders.values()) {
+            cacheRuntimeHandles(painlessClassBuilder);
+        }
+    }
+
+    private void cacheRuntimeHandles(PainlessClassBuilder painlessClassBuilder) {
+        for (PainlessMethod painlessMethod : painlessClassBuilder.methods.values()) {
+            String methodName = painlessMethod.name;
+            int typeParametersSize = painlessMethod.arguments.size();
+
+            if (typeParametersSize == 0 && methodName.startsWith("get") && methodName.length() > 3 &&
+                    Character.isUpperCase(methodName.charAt(3))) {
+                painlessClassBuilder.getters.putIfAbsent(
+                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.handle);
+            } else if (typeParametersSize == 0 && methodName.startsWith("is") && methodName.length() > 2 &&
+                    Character.isUpperCase(methodName.charAt(2))) {
+                painlessClassBuilder.getters.putIfAbsent(
+                        Character.toLowerCase(methodName.charAt(2)) + methodName.substring(3), painlessMethod.handle);
+            } else if (typeParametersSize == 1 && methodName.startsWith("set") && methodName.length() > 3 &&
+                    Character.isUpperCase(methodName.charAt(3))) {
+                painlessClassBuilder.setters.putIfAbsent(
+                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.handle);
+            }
+        }
+
+        for (PainlessField painlessField : painlessClassBuilder.members.values()) {
+            painlessClassBuilder.getters.put(painlessField.name, painlessField.getter);
+            painlessClassBuilder.setters.put(painlessField.name, painlessField.setter);
+        }
+    }
+
+    private void setFunctionalInterfaceMethods() {
+        for (Map.Entry<Class<?>, PainlessClassBuilder> painlessClassBuilderEntry : classesToPainlessClassBuilders.entrySet()) {
+            setFunctionalInterfaceMethod(painlessClassBuilderEntry.getValue());
+        }
+    }
+
+    private void setFunctionalInterfaceMethod(PainlessClassBuilder painlessClassBuilder) {
+        Class<?> targetClass = painlessClassBuilder.clazz;
+
+        if (targetClass.isInterface()) {
+            List<java.lang.reflect.Method> javaMethods = new ArrayList<>();
+
+            for (java.lang.reflect.Method javaMethod : targetClass.getMethods()) {
+                if (javaMethod.isDefault() == false && Modifier.isStatic(javaMethod.getModifiers()) == false) {
+                    try {
+                        Object.class.getMethod(javaMethod.getName(), javaMethod.getParameterTypes());
+                    } catch (ReflectiveOperationException roe) {
+                        javaMethods.add(javaMethod);
                     }
                 }
             }
 
-            // copies methods and fields from super structs to the parent struct
-            copyStruct(painlessStruct.name, painlessSuperStructs);
-
-            // copies methods and fields from Object into interface types
-            if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) {
-                PainlessClassBuilder painlessObjectStruct = classesToPainlessClassBuilders.get(Object.class);
-
-                if (painlessObjectStruct != null) {
-                    copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name));
-                }
+            if (javaMethods.size() != 1 && targetClass.isAnnotationPresent(FunctionalInterface.class)) {
+                throw new IllegalArgumentException("class [" + typeToCanonicalTypeName(targetClass) + "] " +
+                        "is illegally marked as a FunctionalInterface with java methods " + javaMethods);
+            } else if (javaMethods.size() == 1) {
+                java.lang.reflect.Method javaMethod = javaMethods.get(0);
+                String painlessMethodKey = buildPainlessMethodKey(javaMethod.getName(), javaMethod.getParameterCount());
+                painlessClassBuilder.functionalMethod = painlessClassBuilder.methods.get(painlessMethodKey);
             }
         }
-
-        // precompute runtime classes
-        for (PainlessClassBuilder painlessStruct : classesToPainlessClassBuilders.values()) {
-            addRuntimeClass(painlessStruct);
-        }
-
-        Map<Class<?>, PainlessClass> javaClassesToPainlessClasses = new HashMap<>();
-
-        // copy all structs to make them unmodifiable for outside users:
-        for (Map.Entry<Class<?>,PainlessClassBuilder> entry : classesToPainlessClassBuilders.entrySet()) {
-            entry.getValue().functionalMethod = computeFunctionalInterfaceMethod(entry.getValue());
-            javaClassesToPainlessClasses.put(entry.getKey(), entry.getValue().build());
-        }
-
-        return new PainlessLookup(canonicalClassNamesToClasses, javaClassesToPainlessClasses);
     }
 }

From 54ba3ea286478033705d354532351e486bdcd7b2 Mon Sep 17 00:00:00 2001
From: Julie Tibshirani <julie.tibshirani@elastic.co>
Date: Tue, 24 Jul 2018 13:19:24 -0700
Subject: [PATCH 167/260] Undo a debugging change that snuck in during the
 field aliases merge.

---
 .../org/elasticsearch/index/query/MatchQueryBuilderTests.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index 7d72ad9a4e1..cdaccb486f2 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -61,8 +61,8 @@ import static org.hamcrest.Matchers.notNullValue;
 public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuilder> {
     @Override
     protected MatchQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = STRING_ALIAS_FIELD_NAME; //randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME,
-            //INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME,
+            INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
         if (fieldName.equals(DATE_FIELD_NAME)) {
             assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         }

From 717df26fc3e97f4142d3d56ba9fea816799081b0 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Tue, 24 Jul 2018 23:04:22 +0200
Subject: [PATCH 168/260] Networking: Fix test leaking buffer (#32296)

* Test `handler` must release buffer the same way the replaced `org.elasticsearch.http.netty4.Netty4HttpRequestHandler#channelRead0` releases it
* Closes #32289
---
 .../Netty4HttpServerPipeliningTests.java      | 37 +++++++++++--------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
index 3101f660d05..50be7f7ce45 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
@@ -181,27 +181,32 @@ public class Netty4HttpServerPipeliningTests extends ESTestCase {
 
         @Override
         public void run() {
-            final String uri = fullHttpRequest.uri();
+            try {
+                final String uri = fullHttpRequest.uri();
 
-            final ByteBuf buffer = Unpooled.copiedBuffer(uri, StandardCharsets.UTF_8);
+                final ByteBuf buffer = Unpooled.copiedBuffer(uri, StandardCharsets.UTF_8);
 
-            Netty4HttpRequest httpRequest = new Netty4HttpRequest(fullHttpRequest, pipelinedRequest.getSequence());
-            Netty4HttpResponse response = httpRequest.createResponse(RestStatus.OK, new BytesArray(uri.getBytes(StandardCharsets.UTF_8)));
-            response.headers().add(HttpHeaderNames.CONTENT_LENGTH, buffer.readableBytes());
+                Netty4HttpRequest httpRequest = new Netty4HttpRequest(fullHttpRequest, pipelinedRequest.getSequence());
+                Netty4HttpResponse response =
+                    httpRequest.createResponse(RestStatus.OK, new BytesArray(uri.getBytes(StandardCharsets.UTF_8)));
+                response.headers().add(HttpHeaderNames.CONTENT_LENGTH, buffer.readableBytes());
 
-            final boolean slow = uri.matches("/slow/\\d+");
-            if (slow) {
-                try {
-                    Thread.sleep(scaledRandomIntBetween(500, 1000));
-                } catch (InterruptedException e) {
-                    throw new RuntimeException(e);
+                final boolean slow = uri.matches("/slow/\\d+");
+                if (slow) {
+                    try {
+                        Thread.sleep(scaledRandomIntBetween(500, 1000));
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                } else {
+                    assert uri.matches("/\\d+");
                 }
-            } else {
-                assert uri.matches("/\\d+");
-            }
 
-            final ChannelPromise promise = ctx.newPromise();
-            ctx.writeAndFlush(response, promise);
+                final ChannelPromise promise = ctx.newPromise();
+                ctx.writeAndFlush(response, promise);
+            } finally {
+                fullHttpRequest.release();
+            }
         }
 
     }

From e43375bf9abaa7825eec69831d5ba92047896586 Mon Sep 17 00:00:00 2001
From: Jay Modi <jaymode@users.noreply.github.com>
Date: Tue, 24 Jul 2018 16:26:50 -0600
Subject: [PATCH 169/260] Security: revert to old way of merging automata
 (#32254)

This commit reverts to the pre-6.3 way of merging automata as the
change in 6.3 significantly impacts the performance for roles with a
large number of concrete indices. In addition, the maximum number of
states for security automata has been increased to 100,000 in order
to allow users to use roles that caused problems pre-6.3 and 6.3 fixed.

As an escape hatch, the maximum number of states is configurable with
a setting so that users with complex patterns in roles can increase
the states with the knowledge that there is more memory usage.
---
 .../core/security/support/Automatons.java     | 34 ++++++++++++-----
 .../security/support/AutomatonsTests.java     | 38 +++++++++++++++++++
 .../xpack/security/Security.java              |  9 ++---
 3 files changed, 67 insertions(+), 14 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
index 36e0b8ddb00..b11867f8365 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
@@ -9,6 +9,8 @@ import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.RegExp;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -25,9 +27,15 @@ import static org.elasticsearch.common.Strings.collectionToDelimitedString;
 
 public final class Automatons {
 
+    public static final Setting<Integer> MAX_DETERMINIZED_STATES_SETTING =
+        Setting.intSetting("xpack.security.automata.max_determinized_states", 100000, DEFAULT_MAX_DETERMINIZED_STATES,
+            Setting.Property.NodeScope);
     public static final Automaton EMPTY = Automata.makeEmpty();
     public static final Automaton MATCH_ALL = Automata.makeAnyString();
 
+    // this value is not final since we allow it to be set at runtime
+    private static int maxDeterminizedStates = 100000;
+
     static final char WILDCARD_STRING = '*';     // String equality with support for wildcards
     static final char WILDCARD_CHAR = '?';       // Char equality with support for wildcards
     static final char WILDCARD_ESCAPE = '\\';    // Escape character
@@ -49,13 +57,12 @@ public final class Automatons {
         if (patterns.isEmpty()) {
             return EMPTY;
         }
-        Automaton automaton = null;
+        List<Automaton> automata = new ArrayList<>(patterns.size());
         for (String pattern : patterns) {
-            final Automaton patternAutomaton = minimize(pattern(pattern), DEFAULT_MAX_DETERMINIZED_STATES);
-            automaton = automaton == null ? patternAutomaton : unionAndMinimize(Arrays.asList(automaton, patternAutomaton));
+            final Automaton patternAutomaton = pattern(pattern);
+            automata.add(patternAutomaton);
         }
-        // the automaton is always minimized and deterministic
-        return automaton;
+        return unionAndMinimize(automata);
     }
 
     /**
@@ -111,12 +118,12 @@ public final class Automatons {
 
     public static Automaton unionAndMinimize(Collection<Automaton> automata) {
         Automaton res = union(automata);
-        return minimize(res, DEFAULT_MAX_DETERMINIZED_STATES);
+        return minimize(res, maxDeterminizedStates);
     }
 
     public static Automaton minusAndMinimize(Automaton a1, Automaton a2) {
-        Automaton res = minus(a1, a2, DEFAULT_MAX_DETERMINIZED_STATES);
-        return minimize(res, DEFAULT_MAX_DETERMINIZED_STATES);
+        Automaton res = minus(a1, a2, maxDeterminizedStates);
+        return minimize(res, maxDeterminizedStates);
     }
 
     public static Predicate<String> predicate(String... patterns) {
@@ -131,8 +138,17 @@ public final class Automatons {
         return predicate(automaton, "Predicate for " + automaton);
     }
 
+    public static void updateMaxDeterminizedStates(Settings settings) {
+        maxDeterminizedStates = MAX_DETERMINIZED_STATES_SETTING.get(settings);
+    }
+
+    // accessor for testing
+    static int getMaxDeterminizedStates() {
+        return maxDeterminizedStates;
+    }
+
     private static Predicate<String> predicate(Automaton automaton, final String toString) {
-        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, DEFAULT_MAX_DETERMINIZED_STATES);
+        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton, maxDeterminizedStates);
         return new Predicate<String>() {
             @Override
             public boolean test(String s) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
index 92c3b1d7713..72c988fc227 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
@@ -8,8 +8,11 @@ package org.elasticsearch.xpack.core.security.support;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -113,4 +116,39 @@ public class AutomatonsTests extends ESTestCase {
             // expected
         }
     }
+
+    public void testLotsOfIndices() {
+        final int numberOfIndices = scaledRandomIntBetween(512, 1024);
+        final List<String> names = new ArrayList<>(numberOfIndices);
+        for (int i = 0; i < numberOfIndices; i++) {
+            names.add(randomAlphaOfLengthBetween(6, 48));
+        }
+        final Automaton automaton = Automatons.patterns(names);
+        assertTrue(automaton.isDeterministic());
+
+        CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton);
+        for (String name : names) {
+            assertTrue(runAutomaton.run(name));
+        }
+    }
+
+    public void testSettingMaxDeterminizedStates() {
+        try {
+            assertNotEquals(10000, Automatons.getMaxDeterminizedStates());
+            // set to the min value
+            Settings settings = Settings.builder().put(Automatons.MAX_DETERMINIZED_STATES_SETTING.getKey(), 10000).build();
+            Automatons.updateMaxDeterminizedStates(settings);
+            assertEquals(10000, Automatons.getMaxDeterminizedStates());
+
+            final List<String> names = new ArrayList<>(1024);
+            for (int i = 0; i < 1024; i++) {
+                names.add(randomAlphaOfLength(48));
+            }
+            TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class, () -> Automatons.patterns(names));
+            assertThat(e.getMaxDeterminizedStates(), equalTo(10000));
+        } finally {
+            Automatons.updateMaxDeterminizedStates(Settings.EMPTY);
+            assertEquals(100000, Automatons.getMaxDeterminizedStates());
+        }
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index b9cdd22ba47..f4bb4b7eb3b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -126,6 +126,7 @@ import org.elasticsearch.xpack.security.action.privilege.TransportGetPrivilegesA
 import org.elasticsearch.xpack.security.action.privilege.TransportPutPrivilegesAction;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField;
+import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.ssl.SSLConfiguration;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
@@ -217,7 +218,6 @@ import org.elasticsearch.xpack.security.transport.SecurityServerTransportInterce
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport;
 import org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4ServerTransport;
-import org.elasticsearch.xpack.core.template.TemplateUtils;
 import org.elasticsearch.xpack.security.transport.nio.SecurityNioHttpServerTransport;
 import org.elasticsearch.xpack.security.transport.nio.SecurityNioTransport;
 import org.joda.time.DateTime;
@@ -296,9 +296,6 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         this.enabled = XPackSettings.SECURITY_ENABLED.get(settings);
         if (enabled && transportClientMode == false) {
             validateAutoCreateIndex(settings);
-        }
-
-        if (enabled) {
             // we load them all here otherwise we can't access secure settings since they are closed once the checks are
             // fetched
             final List<BootstrapCheck> checks = new ArrayList<>();
@@ -313,6 +310,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 new KerberosRealmBootstrapCheck(env)));
             checks.addAll(InternalRealms.getBootstrapChecks(settings, env));
             this.bootstrapChecks = Collections.unmodifiableList(checks);
+            Automatons.updateMaxDeterminizedStates(settings);
         } else {
             this.bootstrapChecks = Collections.emptyList();
         }
@@ -607,13 +605,14 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         LoggingAuditTrail.registerSettings(settingsList);
         IndexAuditTrail.registerSettings(settingsList);
 
-        // authentication settings
+        // authentication and authorization settings
         AnonymousUser.addSettings(settingsList);
         RealmSettings.addSettings(settingsList, securityExtensions);
         NativeRolesStore.addSettings(settingsList);
         ReservedRealm.addSettings(settingsList);
         AuthenticationService.addSettings(settingsList);
         AuthorizationService.addSettings(settingsList);
+        settingsList.add(Automatons.MAX_DETERMINIZED_STATES_SETTING);
         settingsList.add(CompositeRolesStore.CACHE_SIZE_SETTING);
         settingsList.add(FieldPermissionsCache.CACHE_SIZE_SETTING);
         settingsList.add(TokenService.TOKEN_EXPIRATION);

From 33e3822c5ded4a36742c354608f9ce4220a61029 Mon Sep 17 00:00:00 2001
From: Tim Vernum <tim.vernum@elastic.co>
Date: Wed, 25 Jul 2018 14:14:19 +1000
Subject: [PATCH 170/260] Fix JarHell on X-Pack protocol

Ports 466bc8c from 6.x
---
 x-pack/qa/kerberos-tests/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle
index 5caf5d6947e..2981c0e4ec6 100644
--- a/x-pack/qa/kerberos-tests/build.gradle
+++ b/x-pack/qa/kerberos-tests/build.gradle
@@ -7,7 +7,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
 }

From d0fd23669add7e72e0f344337dc66f6c1c43a2f1 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ikakavas@protonmail.com>
Date: Wed, 25 Jul 2018 08:48:19 +0300
Subject: [PATCH 171/260] Enable FIPS JVM in CI (#32330)

Now that #31666 and #31989 are merged we can run our tests in
fips JVM. This commits enables us to run tests on a Java 8
JVM using BouncyCastleFIPS as a security Provider.
---
 .ci/matrix-runtime-javas.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml
index 737e15ada20..ca7db3b5513 100644
--- a/.ci/matrix-runtime-javas.yml
+++ b/.ci/matrix-runtime-javas.yml
@@ -7,5 +7,6 @@
 
 ES_RUNTIME_JAVA:
   - java8
+  - java8fips
   - java10
   - java11

From b6c14935d725bbe404707ef1d363d8c504a88155 Mon Sep 17 00:00:00 2001
From: Alpar Torok <torokalpar@gmail.com>
Date: Wed, 25 Jul 2018 06:21:03 +0000
Subject: [PATCH 172/260] Determine the minimum gradle version based on the
 wrapper (#32226)

* Determine the minimum gradle version based on the wrapper

This is restrictive and forces users of the plugin to move together with
us, but without integration tests it's close to impossible to make sure
that the claimed compatability is really there.
If we do want to offer more flexibility, we should add those tests
first.

* Track gradle version in individual file

* PR review
---
 build.gradle                                  |  14 ++++++-------
 buildSrc/build.gradle                         |   5 +++--
 .../elasticsearch/gradle/BuildPlugin.groovy   |  19 +++++++++---------
 .../src/main/resources/minimumGradleVersion   |   1 +
 gradle/wrapper/gradle-wrapper.jar             | Bin 54417 -> 54413 bytes
 5 files changed, 20 insertions(+), 19 deletions(-)
 create mode 100644 buildSrc/src/main/resources/minimumGradleVersion

diff --git a/build.gradle b/build.gradle
index 8df89cb064c..3674e0a540b 100644
--- a/build.gradle
+++ b/build.gradle
@@ -16,21 +16,17 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
-import org.apache.tools.ant.taskdefs.condition.Os
-import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
-
-import org.gradle.api.tasks.wrapper.Wrapper
-import org.gradle.api.tasks.wrapper.Wrapper.DistributionType
 import org.gradle.util.GradleVersion
 import org.gradle.util.DistributionLocator
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.apache.tools.ant.filters.ReplaceTokens
 
 import java.nio.file.Files
 import java.nio.file.Path
@@ -562,7 +558,7 @@ task run(type: Run) {
 }
 
 wrapper {
-    distributionType = DistributionType.ALL
+    distributionType = 'ALL'
     doLast {
         final DistributionLocator locator = new DistributionLocator()
         final GradleVersion version = GradleVersion.version(wrapper.gradleVersion)
@@ -571,6 +567,10 @@ wrapper {
         final String sha256Sum = new String(sha256Uri.toURL().bytes)
         wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n"
         println "Added checksum to wrapper properties"
+        // Update build-tools to reflect the Gradle upgrade
+        // TODO: we can remove this once we have tests to make sure older versions work.
+        project(':build-tools').file('src/main/resources/minimumGradleVersion').text = gradleVersion
+        println "Updated minimum Gradle Version"
     }
 }
 
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 55520728812..d507c54c45a 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -25,8 +25,9 @@ plugins {
 
 group = 'org.elasticsearch.gradle'
 
-if (GradleVersion.current() < GradleVersion.version('4.9')) {
-  throw new GradleException('Gradle 4.9+ is required to build elasticsearch')
+String minimumGradleVersion = file('src/main/resources/minimumGradleVersion').text.trim()
+if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion)) {
+    throw new GradleException("Gradle ${minimumGradleVersion}+ is required to build elasticsearch")
 }
 
 if (JavaVersion.current() < JavaVersion.VERSION_1_8) {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 94a8bf26177..5a962a5138b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -20,6 +20,7 @@ package org.elasticsearch.gradle
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
+import org.apache.commons.io.IOUtils
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.eclipse.jgit.lib.Constants
 import org.eclipse.jgit.lib.RepositoryBuilder
@@ -53,6 +54,7 @@ import org.gradle.internal.jvm.Jvm
 import org.gradle.process.ExecResult
 import org.gradle.util.GradleVersion
 
+import java.nio.charset.StandardCharsets
 import java.time.ZoneOffset
 import java.time.ZonedDateTime
 /**
@@ -67,8 +69,13 @@ class BuildPlugin implements Plugin<Project> {
                 + 'elasticearch.standalone-rest-test, and elasticsearch.build '
                 + 'are mutually exclusive')
         }
-        if (GradleVersion.current() < GradleVersion.version('4.9')) {
-            throw new GradleException('Gradle 4.9+ is required to use elasticsearch.build plugin')
+        final String minimumGradleVersion
+        InputStream is = getClass().getResourceAsStream("/minimumGradleVersion")
+        try { minimumGradleVersion = IOUtils.toString(is, StandardCharsets.UTF_8.toString()) } finally { is.close() }
+        if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion.trim())) {
+            throw new GradleException(
+                    "Gradle ${minimumGradleVersion}+ is required to use elasticsearch.build plugin"
+            )
         }
         project.pluginManager.apply('java')
         project.pluginManager.apply('carrotsearch.randomized-testing')
@@ -153,14 +160,6 @@ class BuildPlugin implements Plugin<Project> {
             }
             println "  Random Testing Seed   : ${project.testSeed}"
 
-            // enforce Gradle version
-            final GradleVersion currentGradleVersion = GradleVersion.current();
-
-            final GradleVersion minGradle = GradleVersion.version('4.3')
-            if (currentGradleVersion < minGradle) {
-                throw new GradleException("${minGradle} or above is required to build Elasticsearch")
-            }
-
             // enforce Java version
             if (compilerJavaVersionEnum < minimumCompilerVersion) {
                 final String message =
diff --git a/buildSrc/src/main/resources/minimumGradleVersion b/buildSrc/src/main/resources/minimumGradleVersion
new file mode 100644
index 00000000000..899dd4f5927
--- /dev/null
+++ b/buildSrc/src/main/resources/minimumGradleVersion
@@ -0,0 +1 @@
+4.9
\ No newline at end of file
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index 758de960ec7947253b058ff79c88ce51f3abe08a..0d4a9516871afd710a9d84d89e31ba77745607bd 100644
GIT binary patch
delta 732
zcmYk4Ur19?9LINl)~4m3n;~1*r8tRj;+ldsTG*&j;mD0LiLwYROK2iKWQ|?~A0o3h
zzap4_7J?6pUPQ=BD%&6DHm6Q~$R4bh2%#5!>&Z@s+uDW0{d~^%+{^czdwqed{y>)Z
zc+T&yL!MJAm8!{SYf|rYZFe!1SDt!vF<7`q<#K4UACy<N9_0A}J_f>iSVEQmpTWJ*
zCH4Yyp=$OKJ0T0(LXSg>)8PuzGk9Y~a55RAVIwuGg?b~|_z|w56PmQvMb6Mjj-nRY
z(x#K25gTpjgx1eD(Bo2p{&_ncGbpGYh+64zl`vOT0mdT|v?eK+msKu}NrWG2H3&p&
z=t-@jdDx8_DRo|`ZJ~~CIRy4EIH}`;f_VcmGv8lz>^k${dF(1Rx9~&{Z%aNJQWB2D
zW!4H?+{J?M#ZR(rEX8loqq`_*JA`wIMk;e7)u+cbw-)h)k!CuCwj@e%v@^4%wo?S{
zi*>97FBdP<zHUBIj_;nmaCH}9Hz~oMl*#W=rVF86l`xr<*&ybU?ewKb1dU5hde|%Q
zeXkyyOGcXR<64hbyzZ4{nWmnJbaPmZv1J=g`qKK#Cy7lZ1~Z$@4yG5lSFH4QNYPwu
zte9zRSg0v=mhO)V98Os&IW92YU&0$mD4xiC;r<C88gP5nOk30VHfz8;j-O{an)o3}
ss92L(6!x_{Yy_{@+NdLlr@=DRu1m$4Kcnv77Z&$VxVc`b)`kAcA3t>?lmGw#

delta 674
zcmYk)T}V@590u^ce`_BwX^}IxS%%_;lVr<EjY%_8!l6cqDF{ZiDO5BKTAzy0m1(&i
z69i4_rlboA^P*O%oo(*>!`V#{7~K>D!@P~?^<`NH4(Ing@BclV!#Ts#S?=kqkqXuP
zh4b|_GMTK^ZEaC})BWsS?`2rD8(zFT^*$$2zD;)0rchoVsqNJAp<VohTgt~lsr27Y
zbW5k%1l~wywtzLMlBLlj&qcM@NP8K+__gTs9;Y_BU__<FeYq0f`Km>+Rb5f&VDlMi
zQYES(UnQk?Z>@gyS?N=rh)4VlG+4lCz<faty22Xx{CZ}_s{afX>=!dlOI4&Q6MV!u
zbVzV*se)1`1TBHX<gf`&HD+ThaEzRdBK`<e(?pA4xMdH{290#9UBu_@dU$wj>k#o*
z@F>~s=(iW+T*%107!Fxz!;Wl6Iev$X)Yr|aLv6T;iY{T%osXGt4SR+4a4YGYqG)!i
z`5iSFi0EnGRbCZgAz~)AORzt>9ZfDROi?2}zLBXcG<I8XHCjg>2Lu~p&2;mQ;Ja8o
z&D|9&iPzEWJ<bx0$F20#%~^%ZF%5#F8a4lpnd9x~=FEeKJ24a2U@)Pl*JEOrDJjQ?
zgo&ocGttVfp(okN;z%YhQuY&G7h=Pg3)ixq(zE#NDa72ei6jqanRv6}l%54)O?9zJ
eJWDy46FXK882<m8^8P-DxGth&rBt3TZH7P2bsJXz


From 6aea82956cb9b17481847d83d1881af3e96e3d52 Mon Sep 17 00:00:00 2001
From: Alpar Torok <torokalpar@gmail.com>
Date: Wed, 25 Jul 2018 06:22:54 +0000
Subject: [PATCH 173/260] Number of utilities for writing gradle integration
 tests (#32282)

These are collected from a number of open PRs and are required to
improove existing and write more readable future tests.
I am extracting them to their own PR hoping to be able to merge and use
them sooner.
---
 .../test/GradleIntegrationTestCase.java       | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
index 26da663182f..5c36fa61550 100644
--- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
@@ -1,6 +1,11 @@
 package org.elasticsearch.gradle.test;
 
+import org.gradle.testkit.runner.GradleRunner;
+
 import java.io.File;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
 
@@ -13,4 +18,47 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
         return new File(root, name);
     }
 
+    protected GradleRunner getGradleRunner(String sampleProject) {
+        return GradleRunner.create()
+            .withProjectDir(getProjectDir(sampleProject))
+            .withPluginClasspath();
+    }
+
+    protected File getBuildDir(String name) {
+        return new File(getProjectDir(name), "build");
+    }
+
+    protected void assertOutputContains(String output, String... lines) {
+        for (String line : lines) {
+            assertOutputContains(output, line);
+        }
+        List<Integer> index = Stream.of(lines).map(line -> output.indexOf(line)).collect(Collectors.toList());
+        if (index.equals(index.stream().sorted().collect(Collectors.toList())) == false) {
+            fail("Expected the following lines to appear in this order:\n" +
+                Stream.of(lines).map(line -> "   - `" + line + "`").collect(Collectors.joining("\n")) +
+                "\nBut they did not. Output is:\n\n```" + output + "\n```\n"
+            );
+        }
+    }
+
+    protected void assertOutputContains(String output, String line) {
+        assertTrue(
+            "Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output,
+            output.contains(line)
+        );
+    }
+
+    protected void assertOutputDoesNotContain(String output, String line) {
+        assertFalse(
+            "Expected the following line not to be in output:\n\n" + line + "\n\nOutput is:\n" + output,
+            output.contains(line)
+        );
+    }
+
+    protected void assertOutputDoesNotContain(String output, String... lines) {
+        for (String line : lines) {
+            assertOutputDoesNotContain(line);
+        }
+    }
+
 }

From a2547168d27f0aada27d824f0d7a596a1a2c4fa1 Mon Sep 17 00:00:00 2001
From: Vladimir Dolzhenko <vladimir.dolzhenko@gmail.com>
Date: Wed, 25 Jul 2018 16:34:30 +0200
Subject: [PATCH 174/260] fixes broken build for third-party-tests (#32353)

fixes broken build repository-s3 for third-party-tests
---
 plugins/repository-s3/build.gradle | 44 +++++++++++++++---------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 13119913672..6001ed57065 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -115,7 +115,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
   useFixture = true
 
 } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
-  throw new IllegalArgumentException("not all options specified to run against external S3 service")
+  throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
 }
 
 if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
@@ -126,7 +126,7 @@ if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3T
   s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
 
 } else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
-  throw new IllegalArgumentException("not all options specified to run against external S3 service")
+  throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
 }
 
 final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
@@ -381,31 +381,31 @@ integTestCluster {
 
 integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
 
-///
-RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) {
-  description = "Runs tests using the ECS repository."
-}
+if (useFixture) {
+  RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) {
+    description = "Runs tests using the ECS repository."
+  }
 
 // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
-project.afterEvaluate {
-  ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration
-  cluster.dependsOn(project.s3Fixture)
+  project.afterEvaluate {
+    ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration
+    cluster.dependsOn(project.s3Fixture)
 
-  cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}"
+    cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}"
 
-  Task integTestECSTask = project.tasks.getByName('integTestECS')
-  integTestECSTask.clusterConfig.plugin(project.path)
-  integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
-          "http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint"
-  integTestECSRunner.systemProperty 'tests.rest.blacklist', [
-          'repository_s3/10_basic/*',
-          'repository_s3/20_repository_permanent_credentials/*',
-          'repository_s3/30_repository_temporary_credentials/*',
-          'repository_s3/40_repository_ec2_credentials/*'
-  ].join(",")
+    Task integTestECSTask = project.tasks.getByName('integTestECS')
+    integTestECSTask.clusterConfig.plugin(project.path)
+    integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
+            "http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint"
+    integTestECSRunner.systemProperty 'tests.rest.blacklist', [
+            'repository_s3/10_basic/*',
+            'repository_s3/20_repository_permanent_credentials/*',
+            'repository_s3/30_repository_temporary_credentials/*',
+            'repository_s3/40_repository_ec2_credentials/*'
+    ].join(",")
+  }
+  project.check.dependsOn(integTestECS)
 }
-project.check.dependsOn(integTestECS)
-///
 
 thirdPartyAudit.excludes = [
   // classes are missing

From 9efd1407d5875c34e9f661fd9743c8bd78155878 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Wed, 25 Jul 2018 15:57:07 +0100
Subject: [PATCH 175/260] Add 6.5.0 version to master

This commit adds the 6.5.0 version constant to the master branch.
---
 server/src/main/java/org/elasticsearch/Version.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index da0e3a3b3b1..1fefbad0d74 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -178,6 +178,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_6_3_3 = new Version(V_6_3_3_ID, org.apache.lucene.util.Version.LUCENE_7_3_1);
     public static final int V_6_4_0_ID = 6040099;
     public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
+    public static final int V_6_5_0_ID = 6050099;
+    public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
     public static final Version V_7_0_0_alpha1 =
         new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
@@ -196,6 +198,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
         switch (id) {
             case V_7_0_0_alpha1_ID:
                 return V_7_0_0_alpha1;
+            case V_6_5_0_ID:
+                return V_6_5_0;
             case V_6_4_0_ID:
                 return V_6_4_0;
             case V_6_3_3_ID:

From e12e2e0cdda7745fcfb1570a706d1c8ede21d6e6 Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Wed, 25 Jul 2018 08:07:09 -0700
Subject: [PATCH 176/260] Add opaque_id to index audit logging (#32260)

Logs opaque_id if it is available with all audit log messages using
index-based audit log.

Closes #31521
---
 .../main/resources/security_audit_log.json    |  3 +
 .../security/audit/index/IndexAuditTrail.java |  7 ++
 .../xpack/security/audit/IndexAuditIT.java    | 74 +++++++++++++------
 3 files changed, 62 insertions(+), 22 deletions(-)

diff --git a/x-pack/plugin/core/src/main/resources/security_audit_log.json b/x-pack/plugin/core/src/main/resources/security_audit_log.json
index f5decbb4019..75c25ff53e2 100644
--- a/x-pack/plugin/core/src/main/resources/security_audit_log.json
+++ b/x-pack/plugin/core/src/main/resources/security_audit_log.json
@@ -80,6 +80,9 @@
         },
         "rule": {
           "type": "keyword"
+        },
+        "opaque_id": {
+          "type": "keyword"
         }
       }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
index 1991c2685f2..d8b4b4e4bc1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
@@ -47,6 +47,7 @@ import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.xpack.core.XPackClientPlugin;
@@ -882,6 +883,12 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
         builder.field(Field.NODE_HOST_ADDRESS, nodeHostAddress);
         builder.field(Field.LAYER, layer);
         builder.field(Field.TYPE, type);
+
+        String opaqueId = threadPool.getThreadContext().getHeader(Task.X_OPAQUE_ID);
+        if (opaqueId != null) {
+            builder.field("opaque_id", opaqueId);
+        }
+
         return builder;
     }
 
diff --git a/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java b/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java
index 3467316c24f..a8155b785b3 100644
--- a/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java
+++ b/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java
@@ -11,6 +11,8 @@ import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplat
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
@@ -18,8 +20,10 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.TestCluster;
 import org.elasticsearch.xpack.core.XPackClientPlugin;
@@ -112,8 +116,53 @@ public class IndexAuditIT extends ESIntegTestCase {
                         UsernamePasswordToken.basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()))));
         assertThat(response.getStatusLine().getStatusCode(), is(200));
         final AtomicReference<ClusterState> lastClusterState = new AtomicReference<>();
+        final boolean found = awaitSecurityAuditIndex(lastClusterState, QueryBuilders.matchQuery("principal", USER));
+
+        assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found);
+
+        SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery(
+                QueryBuilders.matchQuery("principal", USER)).get();
+        assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
+        assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("principal"), is(USER));
+    }
+
+    public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception {
+        // this is already "tested" by the test framework since we wipe the templates before and after,
+        // but lets be explicit about the behavior
+        awaitIndexTemplateCreation();
+
+        // delete the template
+        DeleteIndexTemplateResponse deleteResponse = client().admin().indices()
+                .prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
+        assertThat(deleteResponse.isAcknowledged(), is(true));
+        awaitIndexTemplateCreation();
+    }
+
+    public void testOpaqueIdWorking() throws Exception {
+        Request request = new Request("GET", "/");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader(Task.X_OPAQUE_ID, "foo");
+        options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+            UsernamePasswordToken.basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())));
+        request.setOptions(options);
+        Response response = getRestClient().performRequest(request);
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        final AtomicReference<ClusterState> lastClusterState = new AtomicReference<>();
+        final boolean found = awaitSecurityAuditIndex(lastClusterState, QueryBuilders.matchQuery("opaque_id", "foo"));
+
+        assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found);
+
+        SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery(
+            QueryBuilders.matchQuery("opaque_id", "foo")).get();
+        assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
+
+        assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("opaque_id"), is("foo"));
+    }
+
+    private boolean awaitSecurityAuditIndex(AtomicReference<ClusterState> lastClusterState,
+                                            QueryBuilder query) throws InterruptedException {
         final AtomicBoolean indexExists = new AtomicBoolean(false);
-        final boolean found = awaitBusy(() -> {
+        return awaitBusy(() -> {
             if (indexExists.get() == false) {
                 ClusterState state = client().admin().cluster().prepareState().get().getState();
                 lastClusterState.set(state);
@@ -138,28 +187,9 @@ public class IndexAuditIT extends ESIntegTestCase {
             logger.info("refreshing audit indices");
             client().admin().indices().prepareRefresh(".security_audit_log*").get();
             logger.info("refreshed audit indices");
-            return client().prepareSearch(".security_audit_log*").setQuery(QueryBuilders.matchQuery("principal", USER))
-                    .get().getHits().getTotalHits() > 0;
+            return client().prepareSearch(".security_audit_log*").setQuery(query)
+                .get().getHits().getTotalHits() > 0;
         }, 60L, TimeUnit.SECONDS);
-
-        assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found);
-
-        SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery(
-                QueryBuilders.matchQuery("principal", USER)).get();
-        assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
-        assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("principal"), is(USER));
-    }
-
-    public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception {
-        // this is already "tested" by the test framework since we wipe the templates before and after,
-        // but lets be explicit about the behavior
-        awaitIndexTemplateCreation();
-
-        // delete the template
-        DeleteIndexTemplateResponse deleteResponse = client().admin().indices()
-                .prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
-        assertThat(deleteResponse.isAcknowledged(), is(true));
-        awaitIndexTemplateCreation();
     }
 
     private void awaitIndexTemplateCreation() throws InterruptedException {

From 9a7a649755c158977ec238920ed710b993ffceb4 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Wed, 25 Jul 2018 16:10:32 +0100
Subject: [PATCH 177/260] [ML][DOCS] Add documentation for detector rules and
 filters (#32013)

---
 x-pack/docs/build.gradle                      |  10 +
 x-pack/docs/en/ml/api-quickref.asciidoc       |   9 +
 x-pack/docs/en/ml/configuring.asciidoc        |   4 +
 .../docs/en/ml/detector-custom-rules.asciidoc | 230 ++++++++++++++++++
 x-pack/docs/en/ml/functions/geo.asciidoc      |   2 +
 x-pack/docs/en/ml/functions/metric.asciidoc   |   4 +-
 x-pack/docs/en/ml/functions/rare.asciidoc     |   2 +
 x-pack/docs/en/rest-api/defs.asciidoc         |   3 +
 x-pack/docs/en/rest-api/ml-api.asciidoc       |  12 +
 .../en/rest-api/ml/delete-filter.asciidoc     |  53 ++++
 .../en/rest-api/ml/filterresource.asciidoc    |  16 ++
 .../docs/en/rest-api/ml/get-filter.asciidoc   |  84 +++++++
 .../docs/en/rest-api/ml/jobresource.asciidoc  |  73 +++++-
 .../docs/en/rest-api/ml/put-filter.asciidoc   |  68 ++++++
 .../en/rest-api/ml/update-filter.asciidoc     |  67 +++++
 .../docs/en/rest-api/ml/update-job.asciidoc   |  23 +-
 16 files changed, 648 insertions(+), 12 deletions(-)
 create mode 100644 x-pack/docs/en/ml/detector-custom-rules.asciidoc
 create mode 100644 x-pack/docs/en/rest-api/ml/delete-filter.asciidoc
 create mode 100644 x-pack/docs/en/rest-api/ml/filterresource.asciidoc
 create mode 100644 x-pack/docs/en/rest-api/ml/get-filter.asciidoc
 create mode 100644 x-pack/docs/en/rest-api/ml/put-filter.asciidoc
 create mode 100644 x-pack/docs/en/rest-api/ml/update-filter.asciidoc

diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle
index 2ae26044be5..70f6061985a 100644
--- a/x-pack/docs/build.gradle
+++ b/x-pack/docs/build.gradle
@@ -309,6 +309,16 @@ setups['farequote_datafeed'] = setups['farequote_job'] + '''
           "job_id":"farequote",
           "indexes":"farequote"
           }
+'''          
+setups['ml_filter_safe_domains'] = '''
+  - do:
+      xpack.ml.put_filter:
+        filter_id: "safe_domains"
+        body:  >
+          {
+            "description": "A list of safe domains",
+            "items": ["*.google.com", "wikipedia.org"]
+          }
 '''
 setups['server_metrics_index'] = '''
   - do:
diff --git a/x-pack/docs/en/ml/api-quickref.asciidoc b/x-pack/docs/en/ml/api-quickref.asciidoc
index dc87a6ba209..be74167862e 100644
--- a/x-pack/docs/en/ml/api-quickref.asciidoc
+++ b/x-pack/docs/en/ml/api-quickref.asciidoc
@@ -47,6 +47,15 @@ The main {ml} resources can be accessed with a variety of endpoints:
 * {ref}/ml-delete-calendar-job.html[DELETE /calendars/<calendar_id+++>+++/jobs/<job_id+++>+++]: Disassociate a job from a calendar
 * {ref}/ml-delete-calendar.html[DELETE /calendars/<calendar_id+++>+++]: Delete a calendar
 
+[float]
+[[ml-api-filters]]
+=== /filters/
+
+* {ref}/ml-put-filter.html[PUT /filters/<filter_id+++>+++]: Create a filter
+* {ref}/ml-update-filter.html[POST /filters/<filter_id+++>+++/_update]: Update a filter
+* {ref}/ml-get-filter.html[GET /filters/<filter_id+++>+++]: List filters
+* {ref}/ml-delete-filter.html[DELETE /filter/<filter_id+++>+++]: Delete a filter
+
 [float]
 [[ml-api-datafeeds]]
 === /datafeeds/
diff --git a/x-pack/docs/en/ml/configuring.asciidoc b/x-pack/docs/en/ml/configuring.asciidoc
index c2c6e69a711..e35f046a82b 100644
--- a/x-pack/docs/en/ml/configuring.asciidoc
+++ b/x-pack/docs/en/ml/configuring.asciidoc
@@ -34,6 +34,7 @@ The scenarios in this section describe some best practices for generating useful
 * <<ml-configuring-categories>>
 * <<ml-configuring-pop>>
 * <<ml-configuring-transform>>
+* <<ml-configuring-detector-custom-rules>>
 
 :edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/customurl.asciidoc
 include::customurl.asciidoc[]
@@ -49,3 +50,6 @@ include::populations.asciidoc[]
 
 :edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/transforms.asciidoc
 include::transforms.asciidoc[]
+
+:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/detector-custom-rules.asciidoc
+include::detector-custom-rules.asciidoc[]
\ No newline at end of file
diff --git a/x-pack/docs/en/ml/detector-custom-rules.asciidoc b/x-pack/docs/en/ml/detector-custom-rules.asciidoc
new file mode 100644
index 00000000000..18d516fae2d
--- /dev/null
+++ b/x-pack/docs/en/ml/detector-custom-rules.asciidoc
@@ -0,0 +1,230 @@
+[role="xpack"]
+[[ml-configuring-detector-custom-rules]]
+=== Customizing detectors with rules and filters
+
+<<ml-rules,Rules and filters>> enable you to change the behavior of anomaly 
+detectors based on domain-specific knowledge.
+
+Rules describe _when_ a detector should take a certain _action_ instead
+of following its default behavior. To specify the _when_ a rule uses
+a `scope` and `conditions`. You can think of `scope` as the categorical
+specification of a rule, while `conditions` are the numerical part.
+A rule can have a scope, one or more conditions, or a combination of
+scope and conditions.
+
+Let us see how those can be configured by examples.
+
+==== Specifying rule scope
+
+Let us assume we are configuring a job in order to DNS data exfiltration.
+Our data contain fields "subdomain" and "highest_registered_domain".
+We can use a detector that looks like `high_info_content(subdomain) over highest_registered_domain`.
+If we run such a job it is possible that we discover a lot of anomalies on 
+frequently used domains that we have reasons to trust. As security analysts, we 
+are not interested in such anomalies. Ideally, we could instruct the detector to 
+skip results for domains that we consider safe. Using a rule with a scope allows 
+us to achieve this.
+
+First, we need to create a list with our safe domains. Those lists are called 
+`filters` in {ml}. Filters can be shared across jobs.
+
+We create our filter using the {ref}/ml-put-filter.html[put filter API]:
+
+[source,js]
+----------------------------------
+PUT _xpack/ml/filters/safe_domains
+{
+  "description": "Our list of safe domains",
+  "items": ["safe.com", "trusted.com"]
+}
+----------------------------------
+// CONSOLE
+
+Now, we can create our job specifying a scope that uses the filter for the 
+`highest_registered_domain` field:
+
+[source,js]
+----------------------------------
+PUT _xpack/ml/anomaly_detectors/dns_exfiltration_with_rule
+{
+  "analysis_config" : {
+    "bucket_span":"5m",
+    "detectors" :[{
+      "function":"high_info_content",
+      "field_name": "subdomain",
+      "over_field_name": "highest_registered_domain",
+      "custom_rules": [{
+        "actions": ["skip_result"],
+        "scope": {
+          "highest_registered_domain": {
+            "filter_id": "safe_domains",
+            "filter_type": "include"
+          }
+        }
+      }]
+    }]
+  },
+  "data_description" : {
+    "time_field":"timestamp"
+  }
+}
+----------------------------------
+// CONSOLE
+
+As time advances and we see more data and more results, we might encounter new 
+domains that we want to add in the filter. We can do that by using the 
+{ref}/ml-update-filter.html[update filter API]:
+
+[source,js]
+----------------------------------
+POST _xpack/ml/filters/safe_domains/_update
+{
+  "add_items": ["another-safe.com"]
+}
+----------------------------------
+// CONSOLE
+// TEST[setup:ml_filter_safe_domains]
+
+Note that we can provide scope for any of the partition/over/by fields.
+In the following example we scope multiple fields:
+
+[source,js]
+----------------------------------
+PUT _xpack/ml/anomaly_detectors/scoping_multiple_fields
+{
+  "analysis_config" : {
+    "bucket_span":"5m",
+    "detectors" :[{
+      "function":"count",
+      "partition_field_name": "my_partition",
+      "over_field_name": "my_over",
+      "by_field_name": "my_by",
+      "custom_rules": [{
+        "actions": ["skip_result"],
+        "scope": {
+          "my_partition": {
+            "filter_id": "filter_1"
+          },
+          "my_over": {
+            "filter_id": "filter_2"
+          },
+          "my_by": {
+            "filter_id": "filter_3"
+          }
+        }
+      }]
+    }]
+  },
+  "data_description" : {
+    "time_field":"timestamp"
+  }
+}
+----------------------------------
+// CONSOLE
+
+Such a detector will skip results when the values of all 3 scoped fields
+are included in the referenced filters.
+
+==== Specifying rule conditions
+
+Imagine a detector that looks for anomalies in CPU utilization.
+Given a machine that is idle for long enough, small movement in CPU could
+result in anomalous results where the `actual` value is quite small, for 
+example, 0.02. Given our knowledge about how CPU utilization behaves we might 
+determine that anomalies with such small actual values are not interesting for 
+investigation.
+
+Let us now configure a job with a rule that will skip results where CPU 
+utilization is less than 0.20.
+
+[source,js]
+----------------------------------
+PUT _xpack/ml/anomaly_detectors/cpu_with_rule
+{
+  "analysis_config" : {
+    "bucket_span":"5m",
+    "detectors" :[{
+      "function":"high_mean",
+      "field_name": "cpu_utilization",
+      "custom_rules": [{
+        "actions": ["skip_result"],
+        "conditions": [
+          {
+            "applies_to": "actual",
+            "operator": "lt",
+            "value": 0.20
+          }
+        ]
+      }]
+    }]
+  },
+  "data_description" : {
+    "time_field":"timestamp"
+  }
+}
+----------------------------------
+// CONSOLE
+
+When there are multiple conditions they are combined with a logical `and`.
+This is useful when we want the rule to apply to a range. We simply create
+a rule with two conditions, one for each end of the desired range.
+
+Here is an example where a count detector will skip results when the count
+is greater than 30 and less than 50:
+
+[source,js]
+----------------------------------
+PUT _xpack/ml/anomaly_detectors/rule_with_range
+{
+  "analysis_config" : {
+    "bucket_span":"5m",
+    "detectors" :[{
+      "function":"count",
+      "custom_rules": [{
+        "actions": ["skip_result"],
+        "conditions": [
+          {
+            "applies_to": "actual",
+            "operator": "gt",
+            "value": 30
+          },
+          {
+            "applies_to": "actual",
+            "operator": "lt",
+            "value": 50
+          }
+        ]
+      }]
+    }]
+  },
+  "data_description" : {
+    "time_field":"timestamp"
+  }
+}
+----------------------------------
+// CONSOLE
+
+==== Rules in the life-cycle of a job
+
+Rules only affect results created after the rules were applied.
+Let us imagine that we have configured a job and it has been running
+for some time. After observing its results we decide that we can employ
+rules in order to get rid of some uninteresting results. We can use
+the update-job API to do so. However, the rule we added will only be in effect
+for any results created from the moment we added the rule onwards. Past results
+will remain unaffected.
+
+==== Using rules VS filtering data
+
+It might appear like using rules is just another way of filtering the data
+that feeds into a job. For example, a rule that skips results when the
+partition field value is in a filter sounds equivalent to having a query
+that filters out such documents. But it is not. There is a fundamental
+difference. When the data is filtered before reaching a job it is as if they
+never existed for the job. With rules, the data still reaches the job and
+affects its behavior (depending on the rule actions).
+
+For example, a rule with the `skip_result` action means all data will still
+be modeled. On the other hand, a rule with the `skip_model_update` action means
+results will still be created even though the model will not be updated by
+data matched by a rule.
diff --git a/x-pack/docs/en/ml/functions/geo.asciidoc b/x-pack/docs/en/ml/functions/geo.asciidoc
index e9685b46e16..5bcf6c33945 100644
--- a/x-pack/docs/en/ml/functions/geo.asciidoc
+++ b/x-pack/docs/en/ml/functions/geo.asciidoc
@@ -8,6 +8,8 @@ input data.
 The {xpackml} features include the following geographic function: `lat_long`.
 
 NOTE: You cannot create forecasts for jobs that contain geographic functions. 
+You also cannot add rules with conditions to detectors that use geographic 
+functions. 
 
 [float]
 [[ml-lat-long]]
diff --git a/x-pack/docs/en/ml/functions/metric.asciidoc b/x-pack/docs/en/ml/functions/metric.asciidoc
index 3ee51797027..9d6f3515a02 100644
--- a/x-pack/docs/en/ml/functions/metric.asciidoc
+++ b/x-pack/docs/en/ml/functions/metric.asciidoc
@@ -15,6 +15,9 @@ The {xpackml} features include the following metric functions:
 * <<ml-metric-metric,`metric`>>
 * xref:ml-metric-varp[`varp`, `high_varp`, `low_varp`]
 
+NOTE: You cannot add rules with conditions to detectors that use the `metric` 
+function. 
+
 [float]
 [[ml-metric-min]]
 ==== Min
@@ -221,7 +224,6 @@ mean `responsetime` for each application over time. It detects when the mean
 The `metric` function combines `min`, `max`, and `mean` functions. You can use
 it as a shorthand for a combined analysis. If you do not specify a function in
 a detector, this is the default function.
-//TBD: Is that default behavior still true?
 
 High- and low-sided functions are not applicable. You cannot use this function
 when a `summary_count_field_name` is specified.
diff --git a/x-pack/docs/en/ml/functions/rare.asciidoc b/x-pack/docs/en/ml/functions/rare.asciidoc
index fc30918b508..1531285a7ad 100644
--- a/x-pack/docs/en/ml/functions/rare.asciidoc
+++ b/x-pack/docs/en/ml/functions/rare.asciidoc
@@ -15,6 +15,8 @@ number of times (frequency) rare values occur.
 `exclude_frequent`.
 * You cannot create forecasts for jobs that contain `rare` or `freq_rare`
 functions. 
+* You cannot add rules with conditions to detectors that use `rare` or 
+`freq_rare` functions. 
 * Shorter bucket spans (less than 1 hour, for example) are recommended when
 looking for rare events. The functions model whether something happens in a
 bucket at least once. With longer bucket spans, it is more likely that
diff --git a/x-pack/docs/en/rest-api/defs.asciidoc b/x-pack/docs/en/rest-api/defs.asciidoc
index 99600472a09..349ce343c7a 100644
--- a/x-pack/docs/en/rest-api/defs.asciidoc
+++ b/x-pack/docs/en/rest-api/defs.asciidoc
@@ -8,6 +8,7 @@ job configuration options.
 * <<ml-calendar-resource,Calendars>>
 * <<ml-datafeed-resource,{dfeeds-cap}>>
 * <<ml-datafeed-counts,{dfeed-cap} counts>>
+* <<ml-filter-resource,Filters>>
 * <<ml-job-resource,Jobs>>
 * <<ml-jobstats,Job statistics>>
 * <<ml-snapshot-resource,Model snapshots>>
@@ -19,6 +20,8 @@ include::ml/calendarresource.asciidoc[]
 [role="xpack"]
 include::ml/datafeedresource.asciidoc[]
 [role="xpack"]
+include::ml/filterresource.asciidoc[]
+[role="xpack"]
 include::ml/jobresource.asciidoc[]
 [role="xpack"]
 include::ml/jobcounts.asciidoc[]
diff --git a/x-pack/docs/en/rest-api/ml-api.asciidoc b/x-pack/docs/en/rest-api/ml-api.asciidoc
index e9a987cc4a7..b48e9f93404 100644
--- a/x-pack/docs/en/rest-api/ml-api.asciidoc
+++ b/x-pack/docs/en/rest-api/ml-api.asciidoc
@@ -15,6 +15,14 @@ machine learning APIs and in advanced job configuration options in Kibana.
 * <<ml-post-calendar-event,Add scheduled events to calendar>>, <<ml-delete-calendar-event,Delete scheduled events from calendar>>
 * <<ml-get-calendar,Get calendars>>, <<ml-get-calendar-event,Get scheduled events>>
 
+[float]
+[[ml-api-filter-endpoint]]
+=== Filters
+
+* <<ml-put-filter,Create filter>>, <<ml-delete-filter,Delete filter>>
+* <<ml-update-filter,Update filters>>
+* <<ml-get-filter,Get filters>>
+
 [float]
 [[ml-api-datafeed-endpoint]]
 === {dfeeds-cap}
@@ -69,11 +77,13 @@ include::ml/close-job.asciidoc[]
 //CREATE
 include::ml/put-calendar.asciidoc[]
 include::ml/put-datafeed.asciidoc[]
+include::ml/put-filter.asciidoc[]
 include::ml/put-job.asciidoc[]
 //DELETE
 include::ml/delete-calendar.asciidoc[]
 include::ml/delete-datafeed.asciidoc[]
 include::ml/delete-calendar-event.asciidoc[]
+include::ml/delete-filter.asciidoc[]
 include::ml/delete-job.asciidoc[]
 include::ml/delete-calendar-job.asciidoc[]
 include::ml/delete-snapshot.asciidoc[]
@@ -93,6 +103,7 @@ include::ml/get-job.asciidoc[]
 include::ml/get-job-stats.asciidoc[]
 include::ml/get-snapshot.asciidoc[]
 include::ml/get-calendar-event.asciidoc[]
+include::ml/get-filter.asciidoc[]
 include::ml/get-record.asciidoc[]
 //OPEN
 include::ml/open-job.asciidoc[]
@@ -107,6 +118,7 @@ include::ml/start-datafeed.asciidoc[]
 include::ml/stop-datafeed.asciidoc[]
 //UPDATE
 include::ml/update-datafeed.asciidoc[]
+include::ml/update-filter.asciidoc[]
 include::ml/update-job.asciidoc[]
 include::ml/update-snapshot.asciidoc[]
 //VALIDATE
diff --git a/x-pack/docs/en/rest-api/ml/delete-filter.asciidoc b/x-pack/docs/en/rest-api/ml/delete-filter.asciidoc
new file mode 100644
index 00000000000..b58d2980b88
--- /dev/null
+++ b/x-pack/docs/en/rest-api/ml/delete-filter.asciidoc
@@ -0,0 +1,53 @@
+[role="xpack"]
+[[ml-delete-filter]]
+=== Delete Filter API
+++++
+<titleabbrev>Delete Filter</titleabbrev>
+++++
+
+Deletes a filter.
+
+
+==== Request
+
+`DELETE _xpack/ml/filters/<filter_id>`
+
+
+==== Description
+
+This API deletes a {stack-ov}/ml-rules.html[filter]. 
+If a {ml} job references the filter, you cannot delete the filter. You must 
+update or delete the job before you can delete the filter.
+
+
+==== Path Parameters
+
+`filter_id` (required)::
+  (string) Identifier for the filter.
+
+
+==== Authorization
+
+You must have `manage_ml`, or `manage` cluster privileges to use this API.
+For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
+
+
+==== Examples
+
+The following example deletes the `safe_domains` filter:
+
+[source,js]
+--------------------------------------------------
+DELETE _xpack/ml/filters/safe_domains
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:ml_filter_safe_domains]
+
+When the filter is deleted, you receive the following results:
+[source,js]
+----
+{
+  "acknowledged": true
+}
+----
+//TESTRESPONSE
diff --git a/x-pack/docs/en/rest-api/ml/filterresource.asciidoc b/x-pack/docs/en/rest-api/ml/filterresource.asciidoc
new file mode 100644
index 00000000000..64768da4911
--- /dev/null
+++ b/x-pack/docs/en/rest-api/ml/filterresource.asciidoc
@@ -0,0 +1,16 @@
+[role="xpack"]
+[[ml-filter-resource]]
+=== Filter Resources
+
+A filter resource has the following properties:
+
+`filter_id`::
+  (string) A string that uniquely identifies the filter.
+
+`description`::
+  (array) A description of the filter.
+
+`items`::
+  (array of strings) An array of strings which is the filter item list.
+  
+For more information, see {stack-ov}/ml-rules.html[Machine learning rules and filters].
diff --git a/x-pack/docs/en/rest-api/ml/get-filter.asciidoc b/x-pack/docs/en/rest-api/ml/get-filter.asciidoc
new file mode 100644
index 00000000000..89f40cf3312
--- /dev/null
+++ b/x-pack/docs/en/rest-api/ml/get-filter.asciidoc
@@ -0,0 +1,84 @@
+[role="xpack"]
+[[ml-get-filter]]
+=== Get Filters API
+++++
+<titleabbrev>Get Filters</titleabbrev>
+++++
+
+Retrieves filters.
+
+
+==== Request
+
+`GET _xpack/ml/filters/<filter_id>` +
+
+`GET _xpack/ml/filters/`
+
+
+===== Description
+
+You can get a single filter or all filters. For more information, see 
+{stack-ov}/ml-rules.html[Machine learning rules and filters].
+
+
+==== Path Parameters
+
+`filter_id`::
+  (string) Identifier for the filter.
+
+
+==== Request Body
+
+`from`:::
+    (integer) Skips the specified number of filters.
+
+`size`:::
+    (integer) Specifies the maximum number of filters to obtain.
+
+
+==== Results
+
+The API returns the following information:
+
+`filters`::
+  (array) An array of filter resources.
+  For more information, see <<ml-filter-resource>>.
+
+
+==== Authorization
+
+You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
+privileges to use this API. For more information, see
+{xpack-ref}/security-privileges.html[Security Privileges].
+
+
+==== Examples
+
+The following example gets configuration information for the `safe_domains`
+filter:
+
+[source,js]
+--------------------------------------------------
+GET _xpack/ml/filters/safe_domains
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:ml_filter_safe_domains]
+
+The API returns the following results:
+[source,js]
+----
+{
+  "count": 1,
+  "filters": [
+    {
+      "filter_id": "safe_domains",
+      "description": "A list of safe domains",
+      "items": [
+        "*.google.com",
+        "wikipedia.org"
+      ]
+    }
+  ]
+}
+----
+//TESTRESPONSE
diff --git a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
index bb959fd728c..e4b6da11dc5 100644
--- a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
+++ b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
@@ -245,7 +245,7 @@ NOTE: The `field_name` cannot contain double quotes or backslashes.
 --
 
 `function`::
-  (string) The analysis function that is used.
+  (string) The analysis function that is used. 
   For example, `count`, `rare`, `mean`, `min`, `max`, and `sum`. For more
   information, see {xpack-ref}/ml-functions.html[Function Reference].
 
@@ -262,7 +262,12 @@ NOTE: The `field_name` cannot contain double quotes or backslashes.
 
 `use_null`::
   (boolean) Defines whether a new series is used as the null series
-  when there is no value for the by or partition fields. The default value is `false`. +
+  when there is no value for the by or partition fields. The default value is `false`.
+
+`custom_rules`::
+  (array) An array of rule objects, which enable customizing how the detector works.
+  For example, a rule may dictate to the detector conditions under which results should be skipped.
+  For more information see <<ml-detector-custom-rule,detector custom rule objects>>. +
 +
 --
 IMPORTANT: Field names are case sensitive, for example a field named 'Bytes'
@@ -270,9 +275,9 @@ is different from one named 'bytes'.
 
 --
 
-After you create a job, the only property you can change in the detector
-configuration object is the `detector_description`; all other properties are
-informational.
+After you create a job, the only properties you can change in the detector
+configuration object are the `detector_description` and the `custom_rules`;
+all other properties are informational.
 
 [float]
 [[ml-datadescription]]
@@ -408,6 +413,64 @@ the categorization field value came from.
 For more information, see
 {xpack-ref}/ml-configuring-categories.html[Categorizing Log Messages].
 
+[float]
+[[ml-detector-custom-rule]]
+==== Detector Custom Rule
+
+{stack-ov}/ml-rules.html[Custom rules] enable you to customize the way detectors 
+operate. 
+
+A rule has the following properties:
+
+`actions`::
+  (array) The set of actions to be triggered when the rule applies.
+  If more than one action is specified the effects of all actions are combined.
+  The available actions include: +
+  `skip_result`::: The result will not be created. This is the default value.
+  Unless you also specify `skip_model_update`, the model will be updated as
+  usual with the corresponding series value.
+  `skip_model_update`::: The value for that series will not be used to update
+  the model. Unless you also specify `skip_result`, the results will be created
+  as usual. This action is suitable when certain values are expected to be
+  consistently anomalous and they affect the model in a way that negatively
+  impacts the rest of the results.
+`scope`::
+  (object) An optional scope of series where the rule applies. By default the scope
+  includes all series. Scoping is allowed for any of the partition/by/over fields.
+  To add a scope for a field add the field name as a key in the scope object and
+  set its value to an object with properties:
+    `filter_id`::
+      (string) The id of the <<ml-filter-resource,filter>> to be used.
+    `filter_type`::
+      (string) Either `include` (the rule applies for values in the filter)
+      or `exclude` (the rule applies for values not in the filter). Defaults
+      to `include`.
+
+`conditions`::
+  (array) An optional array of numeric conditions when the rule applies.
+  Multiple conditions are combined together with a logical `AND`.
++
+--
+NOTE: If your detector uses `lat_long`, `metric`, `rare`, or `freq_rare` 
+functions, you cannot specify `conditions` for your rule.
+
+  
+A condition has the following properties: 
+
+`applies_to`:::
+    (string) Specifies the result property to which the condition applies.
+    The available options are `actual`, `typical`, `diff_from_typical`, `time`.
+`operator`:::
+    (string) Specifies the condition operator. The available options are
+    `gt` (greater than), `gte` (greater than or equals), `lt` (less than) and `lte` (less than or equals).
+`value`:::
+    (double) The value that is compared against the `applied_to` field using the `operator`.
+--
+
+A rule is required to either have a non-empty scope or at least one condition.
+For more examples see 
+{stack-ov}/ml-configuring-detector-custom-rules.html[Configuring Detector Custom Rules].
+
 [float]
 [[ml-apilimits]]
 ==== Analysis Limits
diff --git a/x-pack/docs/en/rest-api/ml/put-filter.asciidoc b/x-pack/docs/en/rest-api/ml/put-filter.asciidoc
new file mode 100644
index 00000000000..d2982a56f61
--- /dev/null
+++ b/x-pack/docs/en/rest-api/ml/put-filter.asciidoc
@@ -0,0 +1,68 @@
+[role="xpack"]
+[[ml-put-filter]]
+=== Create Filter API
+++++
+<titleabbrev>Create Filter</titleabbrev>
+++++
+
+Instantiates a filter.
+
+==== Request
+
+`PUT _xpack/ml/filters/<filter_id>`
+
+===== Description
+
+A {stack-ov}/ml-rules.html[filter] contains a list of strings. 
+It can be used by one or more jobs. Specifically, filters are referenced in 
+the `custom_rules` property of <<ml-detectorconfig,detector configuration objects>>. 
+
+==== Path Parameters
+
+`filter_id` (required)::
+  (string) Identifier for the filter.
+
+
+==== Request Body
+
+`description`::
+  (string) A description of the filter.
+  
+`items`::
+  (array of strings) The items of the filter.
+  A wildcard `*` can be used at the beginning
+  or the end of an item. Up to 10000 items
+  are allowed in each filter.
+
+
+==== Authorization
+
+You must have `manage_ml`, or `manage` cluster privileges to use this API.
+For more information, see
+{xpack-ref}/security-privileges.html[Security Privileges].
+
+
+==== Examples
+
+The following example creates the `safe_domains` filter:
+
+[source,js]
+--------------------------------------------------
+PUT _xpack/ml/filters/safe_domains
+{
+  "description": "A list of safe domains",
+  "items": ["*.google.com", "wikipedia.org"]
+}
+--------------------------------------------------
+// CONSOLE
+
+When the filter is created, you receive the following response:
+[source,js]
+----
+{
+  "filter_id": "safe_domains",
+  "description": "A list of safe domains",
+  "items": ["*.google.com", "wikipedia.org"]
+}
+----
+//TESTRESPONSE
diff --git a/x-pack/docs/en/rest-api/ml/update-filter.asciidoc b/x-pack/docs/en/rest-api/ml/update-filter.asciidoc
new file mode 100644
index 00000000000..1b6760dfed6
--- /dev/null
+++ b/x-pack/docs/en/rest-api/ml/update-filter.asciidoc
@@ -0,0 +1,67 @@
+[role="xpack"]
+[[ml-update-filter]]
+=== Update Filter API
+++++
+<titleabbrev>Update Filter</titleabbrev>
+++++
+
+Updates the description of a filter, adds items, or removes items. 
+
+==== Request
+
+`POST _xpack/ml/filters/<filter_id>/_update`
+
+//==== Description
+
+==== Path Parameters
+
+`filter_id` (required)::
+		(string) Identifier for the filter.
+
+
+==== Request Body
+
+`description`::
+  (string) A description for the filter. See <<ml-filter-resource>>.
+	
+`add_items`::
+  (array of strings) The items to add to the filter.
+	
+`remove_items`::
+  (array of strings) The items to remove from the filter.
+
+
+==== Authorization
+
+You must have `manage_ml`, or `manage` cluster privileges to use this API.
+For more information, see
+{xpack-ref}/security-privileges.html[Security Privileges].
+
+
+==== Examples
+
+You can change the description, add and remove items to the `safe_domains` filter as follows:
+
+[source,js]
+--------------------------------------------------
+POST _xpack/ml/filters/safe_domains/_update
+{
+  "description": "Updated list of domains",
+  "add_items": ["*.myorg.com"],
+  "remove_items": ["wikipedia.org"]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:ml_filter_safe_domains]
+
+The API returns the following results:
+
+[source,js]
+----
+{
+  "filter_id": "safe_domains",
+  "description": "Updated list of domains",
+  "items": ["*.google.com", "*.myorg.com"]
+}
+----
+//TESTRESPONSE
diff --git a/x-pack/docs/en/rest-api/ml/update-job.asciidoc b/x-pack/docs/en/rest-api/ml/update-job.asciidoc
index 23046febc2e..852745e9dd9 100644
--- a/x-pack/docs/en/rest-api/ml/update-job.asciidoc
+++ b/x-pack/docs/en/rest-api/ml/update-job.asciidoc
@@ -35,6 +35,8 @@ each periodic persistence of the model. See <<ml-job-resource>>. | Yes
 
 |`description` |A description of the job. See <<ml-job-resource>>. | No
 
+|`detectors` |An array of <<ml-detector-update, detector update objects>>. | No
+
 |`groups` |A list of job groups. See <<ml-job-resource>>. | No
 
 |`model_plot_config`: `enabled` |If true, enables calculation and storage of the
@@ -58,12 +60,6 @@ if the job is open when you make the update, you must stop the data feed, close
 the job, then restart the data feed and open the job for the changes to take
 effect.
 
-//|`analysis_config`: `detectors`: `detector_index` | A unique identifier of the
-//detector. Matches the order of detectors returned by
-//<<ml-get-job,GET job>>, starting from 0. | No
-//|`analysis_config`: `detectors`: `detector_description` |A description of the
-//detector. See <<ml-analysisconfig>>. | No
-
 [NOTE]
 --
 * You can update the `analysis_limits` only while the job is closed.
@@ -73,6 +69,21 @@ of `hard_limit`, this means that it was unable to process some data. You might
 want to re-run this job with an increased `model_memory_limit`.
 --
 
+[[ml-detector-update]]
+==== Detector Update Objects
+
+A detector update object has the following properties:
+
+`detector_index`::
+  (integer) The identifier of the detector to update.
+
+`description`::
+  (string) The new description for the detector.
+
+`custom_rules`::
+  (array) The new list of <<ml-detector-custom-rule, rules>> for the detector.
+
+No other detector property can be updated.
 
 ==== Authorization
 

From 6832aa6797c3105747554d9e21213f7d2a15e344 Mon Sep 17 00:00:00 2001
From: lcawl <lcawley@elastic.co>
Date: Tue, 24 Jul 2018 17:06:46 -0700
Subject: [PATCH 178/260] [DOCS] Adds link from bucket_span property to common
 time units

---
 x-pack/docs/en/rest-api/ml/jobresource.asciidoc | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
index e4b6da11dc5..3c4e330722f 100644
--- a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
+++ b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
@@ -106,7 +106,8 @@ An analysis configuration object has the following properties:
 
 `bucket_span`::
   (time units) The size of the interval that the analysis is aggregated into,
-  typically between `5m` and `1h`. The default value is `5m`.
+  typically between `5m` and `1h`. The default value is `5m`. For more 
+  information about time units, see <<time-units,Common options>>.
 
 `categorization_field_name`::
   (string) If this property is specified, the values of the specified field will
@@ -160,8 +161,7 @@ no analysis can occur and an error is returned.
   (time units) The size of the window in which to expect data that is out of
   time order. The default value is 0 (no latency). If you specify a non-zero
   value, it must be greater than or equal to one second. For more information
-  about time units, see
-  {ref}/common-options.html#time-units[Time Units].
+  about time units, see <<time-units,Common options>>.
 +
 --
 NOTE: Latency is only applicable when you send data by using
@@ -511,8 +511,7 @@ Specifying a string is recommended for clarity. If you specify a byte size unit
 of `b` or `kb` and the number does not equate to a discrete number of megabytes,
 it is rounded down to the closest MiB. The minimum valid value is 1 MiB. If you
 specify a value less than 1 MiB, an error occurs. For more information about
-supported byte size units, see
-{ref}/common-options.html#byte-units[Byte size units].
+supported byte size units, see <<byte-units,Common options>>.
 
 If your `elasticsearch.yml` file contains an `xpack.ml.max_model_memory_limit`
 setting, an error occurs when you try to create jobs that have

From 5a12c63b4e98c9ef95baa6cf2e932837e7031420 Mon Sep 17 00:00:00 2001
From: lcawl <lcawley@elastic.co>
Date: Tue, 24 Jul 2018 16:11:56 -0700
Subject: [PATCH 179/260] [DOCS] Fixes typo in ML aggregations page

---
 x-pack/docs/en/ml/aggregations.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/docs/en/ml/aggregations.asciidoc b/x-pack/docs/en/ml/aggregations.asciidoc
index 5ff54b76f01..07f46501569 100644
--- a/x-pack/docs/en/ml/aggregations.asciidoc
+++ b/x-pack/docs/en/ml/aggregations.asciidoc
@@ -105,8 +105,8 @@ For all other aggregations, if the aggregation name doesn't match the field name
 there are limitations in the drill-down functionality within the {ml} page in
 {kib}.
 
-{dfeeds} support complex nested aggregations, this example uses the `derivative`
-pipeline aggregation to find the 1st order derivative of the counter
+{dfeeds-cap} support complex nested aggregations, this example uses the `derivative`
+pipeline aggregation to find the first order derivative of the counter
 `system.network.out.bytes` for each value of the field `beat.name`.
 
 [source,js]

From 6cf7588c3d8e7d79c77c9fd3fd091bc7c11f512d Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Wed, 25 Jul 2018 11:34:26 -0400
Subject: [PATCH 180/260] [TEST] Fix failure due to exception message in java11
 (#32321)

Java 11 uses more verbose exceptions messages, causing this assertion
to fail.  Changed the test to be less restrictive and only look
for the classes we care about.
---
 .../xpack/rollup/RollupResponseTranslationTests.java      | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
index 98e3ad8197a..35d9f0d133a 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java
@@ -508,10 +508,12 @@ public class RollupResponseTranslationTests extends AggregatorTestCase {
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         ScriptService scriptService = mock(ScriptService.class);
         InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(bigArrays, scriptService, true);
-        Exception e = expectThrows(RuntimeException.class,
+        ClassCastException e = expectThrows(ClassCastException.class,
                 () -> RollupResponseTranslator.combineResponses(msearch, reduceContext));
-        assertThat(e.getMessage(), equalTo("org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds " +
-                "cannot be cast to org.elasticsearch.search.aggregations.InternalMultiBucketAggregation"));
+        assertThat(e.getMessage(),
+            containsString("org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds"));
+        assertThat(e.getMessage(),
+            containsString("org.elasticsearch.search.aggregations.InternalMultiBucketAggregation"));
     }
 
     public void testDateHisto() throws IOException {

From de53f0123f99b4cc002f465adcf03142534111b3 Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Wed, 25 Jul 2018 17:35:23 +0100
Subject: [PATCH 181/260] [TEST] Mute
 ConvertProcessortTests.testConvertIntHexError

Relates #32370
---
 .../org/elasticsearch/ingest/common/ConvertProcessorTests.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
index f0fc31dab35..ae27c7850da 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
@@ -67,6 +67,7 @@ public class ConvertProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(10));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
     public void testConvertIntHexError() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String value = "0x" + randomAlphaOfLengthBetween(1, 10);

From 8067250fbe2eed5395f0ed11cac4087bb07053a7 Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Wed, 25 Jul 2018 09:51:40 -0700
Subject: [PATCH 182/260] [DOCS] Adds recommendation for xpack.security.enabled
 (#32345)

---
 docs/reference/settings/security-settings.asciidoc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc
index ba2fc27ed42..4f29b0549b3 100644
--- a/docs/reference/settings/security-settings.asciidoc
+++ b/docs/reference/settings/security-settings.asciidoc
@@ -27,11 +27,17 @@ For more information about creating and updating the {es} keystore, see
 `xpack.security.enabled`::
 Set to `true` to enable {security} on the node. +
 +
+--
 If set to `false`, which is the default value for basic and trial licenses,
 {security} is disabled. It also affects all {kib} instances that connect to this
 {es} instance; you do not need to disable {security} in those `kibana.yml` files.
 For more information about disabling {security} in specific {kib} instances, see  {kibana-ref}/security-settings-kb.html[{kib} security settings].
 
+TIP: If you have gold or higher licenses, the default value is `true`; we 
+recommend that you explicitly add this setting to avoid confusion.  
+
+--
+
 `xpack.security.hide_settings`::
 A comma-separated list of settings that are omitted from the results of the
 <<cluster-nodes-info,cluster nodes info API>>. You can use wildcards to include

From 853aa0afb4b9ac38d30a32ba93f047990367ac44 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Wed, 25 Jul 2018 10:52:01 -0700
Subject: [PATCH 183/260] Painless: Decouple PainlessLookupBuilder and
 Whitelists (#32346)

Implements a static function in PainlessLookupBuilder that contains all the logic related
to Whitelist.  PainlessLookupBuilder is available for use in loading from methods beyond
Whitelist now.
---
 .../painless/PainlessScriptEngine.java        |   4 +-
 .../lookup/PainlessLookupBuilder.java         | 103 ++++++++----------
 .../painless/AnalyzerCasterTests.java         |   1 -
 .../painless/BaseClassTests.java              |  10 +-
 .../painless/BasicStatementTests.java         |   7 +-
 .../elasticsearch/painless/DebugTests.java    |   2 +-
 .../org/elasticsearch/painless/Debugger.java  |   2 +-
 .../painless/DefBootstrapTests.java           |  12 +-
 .../LangPainlessClientYamlTestSuiteIT.java    |   1 -
 .../painless/PainlessDocGenerator.java        |   2 +-
 .../painless/ScriptTestCase.java              |   2 +-
 .../ScriptedMetricAggContextsTests.java       |   2 +-
 .../painless/WhenThingsGoWrongTests.java      |   1 -
 .../painless/node/NodeToStringTests.java      |  18 +--
 14 files changed, 75 insertions(+), 92 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
index 1687cb24cb6..a5a9823d130 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
@@ -103,10 +103,10 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
             ScriptContext<?> context = entry.getKey();
             if (context.instanceClazz.equals(SearchScript.class) || context.instanceClazz.equals(ExecutableScript.class)) {
                 contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class,
-                    new PainlessLookupBuilder(entry.getValue()).build()));
+                        PainlessLookupBuilder.buildFromWhitelists(entry.getValue())));
             } else {
                 contextsToCompilers.put(context, new Compiler(context.instanceClazz,
-                    new PainlessLookupBuilder(entry.getValue()).build()));
+                        PainlessLookupBuilder.buildFromWhitelists(entry.getValue())));
             }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index b15f1f13f20..519227bb901 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -126,14 +126,55 @@ public class PainlessLookupBuilder {
     private static final Pattern METHOD_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][_a-zA-Z0-9]*$");
     private static final Pattern FIELD_NAME_PATTERN  = Pattern.compile("^[_a-zA-Z][_a-zA-Z0-9]*$");
 
-    private final List<Whitelist> whitelists;
+    public static PainlessLookup buildFromWhitelists(List<Whitelist> whitelists) {
+        PainlessLookupBuilder painlessLookupBuilder = new PainlessLookupBuilder();
+        String origin = "internal error";
+
+        try {
+            for (Whitelist whitelist : whitelists) {
+                for (WhitelistClass whitelistClass : whitelist.whitelistStructs) {
+                    origin = whitelistClass.origin;
+                    painlessLookupBuilder.addPainlessClass(
+                            whitelist.javaClassLoader, whitelistClass.javaClassName, whitelistClass.onlyFQNJavaClassName == false);
+                }
+            }
+
+            for (Whitelist whitelist : whitelists) {
+                for (WhitelistClass whitelistClass : whitelist.whitelistStructs) {
+                    String targetCanonicalClassName = whitelistClass.javaClassName.replace('$', '.');
+
+                    for (WhitelistConstructor whitelistConstructor : whitelistClass.whitelistConstructors) {
+                        origin = whitelistConstructor.origin;
+                        painlessLookupBuilder.addPainlessConstructor(
+                                targetCanonicalClassName, whitelistConstructor.painlessParameterTypeNames);
+                    }
+
+                    for (WhitelistMethod whitelistMethod : whitelistClass.whitelistMethods) {
+                        origin = whitelistMethod.origin;
+                        painlessLookupBuilder.addPainlessMethod(
+                                whitelist.javaClassLoader, targetCanonicalClassName, whitelistMethod.javaAugmentedClassName,
+                                whitelistMethod.javaMethodName, whitelistMethod.painlessReturnTypeName,
+                                whitelistMethod.painlessParameterTypeNames);
+                    }
+
+                    for (WhitelistField whitelistField : whitelistClass.whitelistFields) {
+                        origin = whitelistField.origin;
+                        painlessLookupBuilder.addPainlessField(
+                                targetCanonicalClassName, whitelistField.javaFieldName, whitelistField.painlessFieldTypeName);
+                    }
+                }
+            }
+        } catch (Exception exception) {
+            throw new IllegalArgumentException("error loading whitelist(s) " + origin, exception);
+        }
+
+        return painlessLookupBuilder.build();
+    }
 
     private final Map<String, Class<?>> canonicalClassNamesToClasses;
     private final Map<Class<?>, PainlessClassBuilder> classesToPainlessClassBuilders;
 
-    public PainlessLookupBuilder(List<Whitelist> whitelists) {
-        this.whitelists = whitelists;
-
+    public PainlessLookupBuilder() {
         canonicalClassNamesToClasses = new HashMap<>();
         classesToPainlessClassBuilders = new HashMap<>();
 
@@ -666,60 +707,6 @@ public class PainlessLookupBuilder {
     }
 
     public PainlessLookup build() {
-        String origin = "internal error";
-
-        try {
-            // first iteration collects all the Painless type names that
-            // are used for validation during the second iteration
-            for (Whitelist whitelist : whitelists) {
-                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
-                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-                    PainlessClassBuilder painlessStruct =
-                            classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(painlessTypeName));
-
-                    if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
-                        throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
-                                "[" + painlessStruct.clazz.getName() + "] and [" + whitelistStruct.javaClassName + "]");
-                    }
-
-                    origin = whitelistStruct.origin;
-                    addPainlessClass(
-                            whitelist.javaClassLoader, whitelistStruct.javaClassName, whitelistStruct.onlyFQNJavaClassName == false);
-
-                    painlessStruct = classesToPainlessClassBuilders.get(canonicalClassNamesToClasses.get(painlessTypeName));
-                    classesToPainlessClassBuilders.put(painlessStruct.clazz, painlessStruct);
-                }
-            }
-
-            // second iteration adds all the constructors, methods, and fields that will
-            // be available in Painless along with validating they exist and all their types have
-            // been white-listed during the first iteration
-            for (Whitelist whitelist : whitelists) {
-                for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) {
-                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
-
-                    for (WhitelistConstructor whitelistConstructor : whitelistStruct.whitelistConstructors) {
-                        origin = whitelistConstructor.origin;
-                        addPainlessConstructor(painlessTypeName, whitelistConstructor.painlessParameterTypeNames);
-                    }
-
-                    for (WhitelistMethod whitelistMethod : whitelistStruct.whitelistMethods) {
-                        origin = whitelistMethod.origin;
-                        addPainlessMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod.javaAugmentedClassName,
-                                whitelistMethod.javaMethodName, whitelistMethod.painlessReturnTypeName,
-                                whitelistMethod.painlessParameterTypeNames);
-                    }
-
-                    for (WhitelistField whitelistField : whitelistStruct.whitelistFields) {
-                        origin = whitelistField.origin;
-                        addPainlessField(painlessTypeName, whitelistField.javaFieldName, whitelistField.painlessFieldTypeName);
-                    }
-                }
-            }
-        } catch (Exception exception) {
-            throw new IllegalArgumentException("error loading whitelist(s) " + origin, exception);
-        }
-
         copyPainlessClassMembers();
         cacheRuntimeHandles();
         setFunctionalInterfaceMethods();
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java
index a702490fff9..34bc2c78de6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessCast;
-
 import org.elasticsearch.test.ESTestCase;
 
 public class AnalyzerCasterTests extends ESTestCase {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
index c0e0bd7ed9d..c852d5a41de 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
@@ -19,14 +19,14 @@
 
 package org.elasticsearch.painless;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.spi.Whitelist;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.containsString;
@@ -38,7 +38,7 @@ import static org.hamcrest.Matchers.startsWith;
  */
 public class BaseClassTests extends ScriptTestCase {
 
-    private final PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
+    private final PainlessLookup painlessLookup = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
 
     public abstract static class Gets {
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index e397013e06f..0f5844c6599 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -2,6 +2,9 @@ package org.elasticsearch.painless;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /*
  * Licensed to Elasticsearch under one or more contributor
@@ -22,10 +25,6 @@ import java.util.Collections;
  * under the License.
  */
 
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 public class BasicStatementTests extends ScriptTestCase {
 
     public void testIfStatement() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
index 7edc90bb0a0..f0f784c089a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
@@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.not;
 
 public class DebugTests extends ScriptTestCase {
-    private final PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
+    private final PainlessLookup painlessLookup = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
 
     public void testExplain() {
         // Debug.explain can explain an object
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index 73adf92779d..48af3898e09 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -40,7 +40,7 @@ final class Debugger {
         PrintWriter outputWriter = new PrintWriter(output);
         Textifier textifier = new Textifier();
         try {
-            new Compiler(iface, new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build())
+            new Compiler(iface, PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS))
                     .compile("<debugging>", source, settings, textifier);
         } catch (RuntimeException e) {
             textifier.print(outputWriter);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 07f45ff67c0..1ef855d561c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.test.ESTestCase;
+
 import java.lang.invoke.CallSite;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
@@ -27,13 +32,8 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
-import org.elasticsearch.painless.spi.Whitelist;
-import org.elasticsearch.test.ESTestCase;
-
 public class DefBootstrapTests extends ESTestCase {
-    private final PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
+    private final PainlessLookup painlessLookup = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
 
     /** calls toString() on integers, twice */
     public void testOneType() throws Throwable {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
index 2d3f299bbcb..1ab53b4e430 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.painless;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index e26a5a38c76..b2cc5e48ad8 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -52,7 +52,7 @@ import static java.util.stream.Collectors.toList;
  */
 public class PainlessDocGenerator {
 
-    private static final PainlessLookup PAINLESS_LOOKUP = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
+    private static final PainlessLookup PAINLESS_LOOKUP = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
     private static final Logger logger = ESLoggerFactory.getLogger(PainlessDocGenerator.class);
     private static final Comparator<PainlessField> FIELD_NAME = comparing(f -> f.name);
     private static final Comparator<PainlessMethod> METHOD_NAME = comparing(m -> m.name);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index eebf1d701ee..96cc296a1af 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -92,7 +92,7 @@ public abstract class ScriptTestCase extends ESTestCase {
     public Object exec(String script, Map<String, Object> vars, Map<String,String> compileParams, Scorer scorer, boolean picky) {
         // test for ambiguity errors before running the actual script if picky is true
         if (picky) {
-            PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
+            PainlessLookup painlessLookup = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
             ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, GenericElasticsearchScript.class);
             CompilerSettings pickySettings = new CompilerSettings();
             pickySettings.setPicky(true);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
index 66d49be16ba..6ee021c695f 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
@@ -22,8 +22,8 @@ package org.elasticsearch.painless;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.painless.spi.Whitelist;
-import org.elasticsearch.script.ScriptedMetricAggContexts;
 import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptedMetricAggContexts;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 1bb754db847..8eeb25c9676 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless;
 
 import junit.framework.AssertionFailedError;
-
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.script.ScriptException;
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index cd3e4123e12..84452b4843d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -20,21 +20,21 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessCast;
-import org.elasticsearch.painless.lookup.PainlessField;
-import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.FeatureTest;
 import org.elasticsearch.painless.GenericElasticsearchScript;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.ScriptClassInfo;
-import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.painless.antlr.Walker;
+import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessField;
+import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
+import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -49,7 +49,7 @@ import static org.elasticsearch.painless.node.SSource.MainMethodReserved;
  * Tests {@link Object#toString} implementations on all extensions of {@link ANode}.
  */
 public class NodeToStringTests extends ESTestCase {
-    private final PainlessLookup painlessLookup = new PainlessLookupBuilder(Whitelist.BASE_WHITELISTS).build();
+    private final PainlessLookup painlessLookup = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
 
     public void testEAssignment() {
         assertToString(

From 10bfedeb530f2ad082ddaf74a8efc3fc45644b1b Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Wed, 25 Jul 2018 10:59:25 -0700
Subject: [PATCH 184/260] Painless: Fix documentation links to use existing
 refs (#32335)

Uses ref and xpack-ref instead of custom ones in the Painless docs for
long-term maintainability.
---
 docs/painless/painless-contexts.asciidoc      | 41 +++++++++----------
 .../painless-bucket-agg-context.asciidoc      |  2 +-
 .../painless-field-context.asciidoc           |  4 +-
 .../painless-filter-context.asciidoc          |  2 +-
 ...painless-ingest-processor-context.asciidoc | 10 ++---
 ...inless-metric-agg-combine-context.asciidoc |  2 +-
 .../painless-metric-agg-init-context.asciidoc |  2 +-
 .../painless-metric-agg-map-context.asciidoc  |  2 +-
 ...ainless-metric-agg-reduce-context.asciidoc |  2 +-
 ...painless-min-should-match-context.asciidoc |  2 +-
 .../painless-reindex-context.asciidoc         | 22 +++++-----
 .../painless-score-context.asciidoc           |  2 +-
 .../painless-similarity-context.asciidoc      |  2 +-
 .../painless-sort-context.asciidoc            |  2 +-
 .../painless-update-by-query-context.asciidoc | 14 +++----
 .../painless-update-context.asciidoc          | 14 +++----
 ...ainless-watcher-condition-context.asciidoc |  4 +-
 ...ainless-watcher-transform-context.asciidoc |  4 +-
 .../painless-weight-context.asciidoc          |  2 +-
 19 files changed, 66 insertions(+), 69 deletions(-)

diff --git a/docs/painless/painless-contexts.asciidoc b/docs/painless/painless-contexts.asciidoc
index ff46f6bd74d..8b8a3b0eec6 100644
--- a/docs/painless/painless-contexts.asciidoc
+++ b/docs/painless/painless-contexts.asciidoc
@@ -1,9 +1,6 @@
 [[painless-contexts]]
 == Painless contexts
 
-:es_version: https://www.elastic.co/guide/en/elasticsearch/reference/master
-:xp_version: https://www.elastic.co/guide/en/x-pack/current
-
 A Painless script is evaluated within a context. Each context has values that
 are available as local variables, a whitelist that controls the available
 classes, and the methods and fields within those classes (API), and
@@ -18,41 +15,41 @@ specialized code may define new ways to use a Painless script.
 | Name                              | Painless Documentation
                                     | Elasticsearch Documentation
 | Update                            | <<painless-update-context, Painless Documentation>>
-                                    | {es_version}/docs-update.html[Elasticsearch Documentation]
+                                    | {ref}/docs-update.html[Elasticsearch Documentation]
 | Update by query                   | <<painless-update-by-query-context, Painless Documentation>>
-                                    | {es_version}/docs-update-by-query.html[Elasticsearch Documentation]
+                                    | {ref}/docs-update-by-query.html[Elasticsearch Documentation]
 | Reindex                           | <<painless-reindex-context, Painless Documentation>>
-                                    | {es_version}/docs-reindex.html[Elasticsearch Documentation]
+                                    | {ref}/docs-reindex.html[Elasticsearch Documentation]
 | Sort                              | <<painless-sort-context, Painless Documentation>>
-                                    | {es_version}/search-request-sort.html[Elasticsearch Documentation]
+                                    | {ref}/search-request-sort.html[Elasticsearch Documentation]
 | Similarity                        | <<painless-similarity-context, Painless Documentation>>
-                                    | {es_version}/index-modules-similarity.html[Elasticsearch Documentation]
-| Weight                            | <<painless-similarity-context, Painless Documentation>>
-                                    | {es_version}/index-modules-similarity.html[Elasticsearch Documentation]
+                                    | {ref}/index-modules-similarity.html[Elasticsearch Documentation]
+| Weight                            | <<painless-weight-context, Painless Documentation>>
+                                    | {ref}/index-modules-similarity.html[Elasticsearch Documentation]
 | Score                             | <<painless-score-context, Painless Documentation>>
-                                    | {es_version}/query-dsl-function-score-query.html[Elasticsearch Documentation]
+                                    | {ref}/query-dsl-function-score-query.html[Elasticsearch Documentation]
 | Field                             | <<painless-field-context, Painless Documentation>>
-                                    | {es_version}/search-request-script-fields.html[Elasticsearch Documentation]
+                                    | {ref}/search-request-script-fields.html[Elasticsearch Documentation]
 | Filter                            | <<painless-filter-context, Painless Documentation>>
-                                    | {es_version}/query-dsl-script-query.html[Elasticsearch Documentation]
+                                    | {ref}/query-dsl-script-query.html[Elasticsearch Documentation]
 | Minimum should match              | <<painless-min-should-match-context, Painless Documentation>>
-                                    | {es_version}/query-dsl-terms-set-query.html[Elasticsearch Documentation]
+                                    | {ref}/query-dsl-terms-set-query.html[Elasticsearch Documentation]
 | Metric aggregation initialization | <<painless-metric-agg-init-context, Painless Documentation>>
-                                    | {es_version}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
+                                    | {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
 | Metric aggregation map            | <<painless-metric-agg-map-context, Painless Documentation>>
-                                    | {es_version}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
+                                    | {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
 | Metric aggregation combine        | <<painless-metric-agg-combine-context, Painless Documentation>>
-                                    | {es_version}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
+                                    | {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
 | Metric aggregation reduce         | <<painless-metric-agg-reduce-context, Painless Documentation>>
-                                    | {es_version}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
+                                    | {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
 | Bucket aggregation                | <<painless-bucket-agg-context, Painless Documentation>>
-                                    | {es_version}/search-aggregations-pipeline-bucket-script-aggregation.html[Elasticsearch Documentation]
+                                    | {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[Elasticsearch Documentation]
 | Ingest processor                  | <<painless-ingest-processor-context, Painless Documentation>>
-                                    | {es_version}/script-processor.html[Elasticsearch Documentation]
+                                    | {ref}/script-processor.html[Elasticsearch Documentation]
 | Watcher condition                 | <<painless-watcher-condition-context, Painless Documentation>>
-                                    | {xp_version}/condition-script.html[Elasticsearch Documentation]
+                                    | {xpack-ref}/condition-script.html[Elasticsearch Documentation]
 | Watcher transform                 | <<painless-watcher-transform-context, Painless Documentation>>
-                                    | {xp_version}/transform-script.html[Elasticsearch Documentation]
+                                    | {xpack-ref}/transform-script.html[Elasticsearch Documentation]
 |====
 
 include::painless-contexts/index.asciidoc[]
diff --git a/docs/painless/painless-contexts/painless-bucket-agg-context.asciidoc b/docs/painless/painless-contexts/painless-bucket-agg-context.asciidoc
index b277055d87d..3bb4cae3d3b 100644
--- a/docs/painless/painless-contexts/painless-bucket-agg-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-bucket-agg-context.asciidoc
@@ -2,7 +2,7 @@
 === Bucket aggregation context
 
 Use a Painless script in an
-{es_version}/search-aggregations-pipeline-bucket-script-aggregation.html[bucket aggregation]
+{ref}/search-aggregations-pipeline-bucket-script-aggregation.html[bucket aggregation]
 to calculate a value as a result in a bucket.
 
 *Variables*
diff --git a/docs/painless/painless-contexts/painless-field-context.asciidoc b/docs/painless/painless-contexts/painless-field-context.asciidoc
index bf44703001b..4c767ca3891 100644
--- a/docs/painless/painless-contexts/painless-field-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-field-context.asciidoc
@@ -2,7 +2,7 @@
 === Field context
 
 Use a Painless script to create a
-{es_version}/search-request-script-fields.html[script field] to return
+{ref}/search-request-script-fields.html[script field] to return
 a customized value for each document in the results of a query.
 
 *Variables*
@@ -14,7 +14,7 @@ a customized value for each document in the results of a query.
         Contains the fields of the specified document where each field is a
         `List` of values.
 
-{es_version}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
+{ref}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
         Contains extracted JSON in a `Map` and `List` structure for the fields
         existing in a stored document.
 
diff --git a/docs/painless/painless-contexts/painless-filter-context.asciidoc b/docs/painless/painless-contexts/painless-filter-context.asciidoc
index ea0393893c8..96fddf13b50 100644
--- a/docs/painless/painless-contexts/painless-filter-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-filter-context.asciidoc
@@ -1,7 +1,7 @@
 [[painless-filter-context]]
 === Filter context
 
-Use a Painless script as a {es_version}/query-dsl-script-query.html[filter] in a
+Use a Painless script as a {ref}/query-dsl-script-query.html[filter] in a
 query to include and exclude documents.
 
 
diff --git a/docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc b/docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc
index ba3be073963..5d451268ded 100644
--- a/docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-ingest-processor-context.asciidoc
@@ -1,7 +1,7 @@
 [[painless-ingest-processor-context]]
 === Ingest processor context
 
-Use a Painless script in an {es_version}/script-processor.html[ingest processor]
+Use a Painless script in an {ref}/script-processor.html[ingest processor]
 to modify documents upon insertion.
 
 *Variables*
@@ -9,10 +9,10 @@ to modify documents upon insertion.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-{es_version}/mapping-index-field.html[`ctx['_index']`] (`String`)::
+{ref}/mapping-index-field.html[`ctx['_index']`] (`String`)::
         The name of the index.
 
-{es_version}/mapping-type-field.html[`ctx['_type']`] (`String`)::
+{ref}/mapping-type-field.html[`ctx['_type']`] (`String`)::
         The type of document within an index.
 
 `ctx` (`Map`)::
@@ -21,10 +21,10 @@ to modify documents upon insertion.
 
 *Side Effects*
 
-{es_version}/mapping-index-field.html[`ctx['_index']`]::
+{ref}/mapping-index-field.html[`ctx['_index']`]::
         Modify this to change the destination index for the current document.
 
-{es_version}/mapping-type-field.html[`ctx['_type']`]::
+{ref}/mapping-type-field.html[`ctx['_type']`]::
         Modify this to change the type for the current document.
 
 `ctx` (`Map`, read-only)::
diff --git a/docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc
index 1fec63ef446..31cb596ae81 100644
--- a/docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-metric-agg-combine-context.asciidoc
@@ -2,7 +2,7 @@
 === Metric aggregation combine context
 
 Use a Painless script to
-{es_version}/search-aggregations-metrics-scripted-metric-aggregation.html[combine]
+{ref}/search-aggregations-metrics-scripted-metric-aggregation.html[combine]
 values for use in a scripted metric aggregation. A combine script is run once
 per shard following a <<painless-metric-agg-map-context, map script>> and is
 optional as part of a full metric aggregation.
diff --git a/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc
index ed7e01ddd00..1503e3abb58 100644
--- a/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc
@@ -2,7 +2,7 @@
 === Metric aggregation initialization context
 
 Use a Painless script to
-{es_version}/search-aggregations-metrics-scripted-metric-aggregation.html[initialize]
+{ref}/search-aggregations-metrics-scripted-metric-aggregation.html[initialize]
 values for use in a scripted metric aggregation. An initialization script is
 run prior to document collection once per shard and is optional as part of the
 full metric aggregation.
diff --git a/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc
index 51f06e010db..16016d1cf81 100644
--- a/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc
@@ -2,7 +2,7 @@
 === Metric aggregation map context
 
 Use a Painless script to
-{es_version}/search-aggregations-metrics-scripted-metric-aggregation.html[map]
+{ref}/search-aggregations-metrics-scripted-metric-aggregation.html[map]
 values for use in a scripted metric aggregation. A map script is run once per
 collected document following an optional
 <<painless-metric-agg-init-context, initialization script>> and is required as
diff --git a/docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc
index 1b64b85392d..b76e02b1b04 100644
--- a/docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-metric-agg-reduce-context.asciidoc
@@ -2,7 +2,7 @@
 === Metric aggregation reduce context
 
 Use a Painless script to
-{es_version}/search-aggregations-metrics-scripted-metric-aggregation.html[reduce]
+{ref}/search-aggregations-metrics-scripted-metric-aggregation.html[reduce]
 values to produce the result of a scripted metric aggregation. A reduce script
 is run once on the coordinating node following a
 <<painless-metric-agg-combine-context, combine script>> (or a
diff --git a/docs/painless/painless-contexts/painless-min-should-match-context.asciidoc b/docs/painless/painless-contexts/painless-min-should-match-context.asciidoc
index c310f42928e..b2ffb63fd7a 100644
--- a/docs/painless/painless-contexts/painless-min-should-match-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-min-should-match-context.asciidoc
@@ -2,7 +2,7 @@
 === Minimum should match context
 
 Use a Painless script to specify the
-{es_version}/query-dsl-terms-set-query.html[minimum] number of terms that a
+{ref}/query-dsl-terms-set-query.html[minimum] number of terms that a
 specified field needs to match with for a document to be part of the query
 results.
 
diff --git a/docs/painless/painless-contexts/painless-reindex-context.asciidoc b/docs/painless/painless-contexts/painless-reindex-context.asciidoc
index a8477c8c619..ae5445183a6 100644
--- a/docs/painless/painless-contexts/painless-reindex-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-reindex-context.asciidoc
@@ -1,7 +1,7 @@
 [[painless-reindex-context]]
 === Reindex context
 
-Use a Painless script in a {es_version}/docs-reindex.html[reindex] operation to
+Use a Painless script in a {ref}/docs-reindex.html[reindex] operation to
 add, modify, or delete fields within each document in an original index as its
 reindexed into a target index.
 
@@ -13,22 +13,22 @@ reindexed into a target index.
 `ctx['_op']` (`String`)::
         The name of the operation.
 
-{es_version}/mapping-routing-field.html[`ctx['_routing']`] (`String`)::
+{ref}/mapping-routing-field.html[`ctx['_routing']`] (`String`)::
         The value used to select a shard for document storage.
 
-{es_version}/mapping-index-field.html[`ctx['_index']`] (`String`)::
+{ref}/mapping-index-field.html[`ctx['_index']`] (`String`)::
         The name of the index.
 
-{es_version}/mapping-type-field.html[`ctx['_type']`] (`String`)::
+{ref}/mapping-type-field.html[`ctx['_type']`] (`String`)::
         The type of document within an index.
 
-{es_version}/mapping-id-field.html[`ctx['_id']`] (`int`, read-only)::
+{ref}/mapping-id-field.html[`ctx['_id']`] (`int`, read-only)::
         The unique document id.
 
 `ctx['_version']` (`int`)::
         The current version of the document.
 
-{es_version}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
+{ref}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
         Contains extracted JSON in a `Map` and `List` structure for the fields
         existing in a stored document.
 
@@ -39,22 +39,22 @@ reindexed into a target index.
         specify no operation or `delete` to delete the current document from
         the index.
 
-{es_version}/mapping-routing-field.html[`ctx['_routing']`]::
+{ref}/mapping-routing-field.html[`ctx['_routing']`]::
         Modify this to change the routing value for the current document.
 
-{es_version}/mapping-index-field.html[`ctx['_index']`]::
+{ref}/mapping-index-field.html[`ctx['_index']`]::
         Modify this to change the destination index for the current document.
 
-{es_version}/mapping-type-field.html[`ctx['_type']`]::
+{ref}/mapping-type-field.html[`ctx['_type']`]::
         Modify this to change the type for the current document.
 
-{es_version}/mapping-id-field.html[`ctx['_id']`]::
+{ref}/mapping-id-field.html[`ctx['_id']`]::
         Modify this to change the id for the current document.
 
 `ctx['_version']` (`int`)::
         Modify this to modify the version for the current document.
 
-{es_version}/mapping-source-field.html[`ctx['_source']`]::
+{ref}/mapping-source-field.html[`ctx['_source']`]::
         Modify the values in the `Map/List` structure to add, modify, or delete
         the fields of a document.
 
diff --git a/docs/painless/painless-contexts/painless-score-context.asciidoc b/docs/painless/painless-contexts/painless-score-context.asciidoc
index 21667fd31f3..bd1e1de7f77 100644
--- a/docs/painless/painless-contexts/painless-score-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-score-context.asciidoc
@@ -2,7 +2,7 @@
 === Score context
 
 Use a Painless script in a
-{es_version}/query-dsl-function-score-query.html[function score] to apply a new
+{ref}/query-dsl-function-score-query.html[function score] to apply a new
 score to documents returned from a query.
 
 *Variables*
diff --git a/docs/painless/painless-contexts/painless-similarity-context.asciidoc b/docs/painless/painless-contexts/painless-similarity-context.asciidoc
index 052844c3111..53b37be52b6 100644
--- a/docs/painless/painless-contexts/painless-similarity-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-similarity-context.asciidoc
@@ -2,7 +2,7 @@
 === Similarity context
 
 Use a Painless script to create a
-{es_version}/index-modules-similarity.html[similarity] equation for scoring
+{ref}/index-modules-similarity.html[similarity] equation for scoring
 documents in a query.
 
 *Variables*
diff --git a/docs/painless/painless-contexts/painless-sort-context.asciidoc b/docs/painless/painless-contexts/painless-sort-context.asciidoc
index 7f510fb6a92..9efd5076688 100644
--- a/docs/painless/painless-contexts/painless-sort-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-sort-context.asciidoc
@@ -2,7 +2,7 @@
 === Sort context
 
 Use a Painless script to
-{es_version}/search-request-sort.html[sort] the documents in a query.
+{ref}/search-request-sort.html[sort] the documents in a query.
 
 *Variables*
 
diff --git a/docs/painless/painless-contexts/painless-update-by-query-context.asciidoc b/docs/painless/painless-contexts/painless-update-by-query-context.asciidoc
index 65666e15844..d8d77548074 100644
--- a/docs/painless/painless-contexts/painless-update-by-query-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-update-by-query-context.asciidoc
@@ -2,7 +2,7 @@
 === Update by query context
 
 Use a Painless script in an
-{es_version}/docs-update-by-query.html[update by query] operation to add,
+{ref}/docs-update-by-query.html[update by query] operation to add,
 modify, or delete fields within each of a set of documents collected as the
 result of query.
 
@@ -14,22 +14,22 @@ result of query.
 `ctx['_op']` (`String`)::
         The name of the operation.
 
-{es_version}/mapping-routing-field.html[`ctx['_routing']`] (`String`, read-only)::
+{ref}/mapping-routing-field.html[`ctx['_routing']`] (`String`, read-only)::
         The value used to select a shard for document storage.
 
-{es_version}/mapping-index-field.html[`ctx['_index']`] (`String`, read-only)::
+{ref}/mapping-index-field.html[`ctx['_index']`] (`String`, read-only)::
         The name of the index.
 
-{es_version}/mapping-type-field.html[`ctx['_type']`] (`String`, read-only)::
+{ref}/mapping-type-field.html[`ctx['_type']`] (`String`, read-only)::
         The type of document within an index.
 
-{es_version}/mapping-id-field.html[`ctx['_id']`] (`int`, read-only)::
+{ref}/mapping-id-field.html[`ctx['_id']`] (`int`, read-only)::
         The unique document id.
 
 `ctx['_version']` (`int`, read-only)::
         The current version of the document.
 
-{es_version}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
+{ref}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
         Contains extracted JSON in a `Map` and `List` structure for the fields
         existing in a stored document.
 
@@ -40,7 +40,7 @@ result of query.
         specify no operation or `delete` to delete the current document from
         the index.
 
-{es_version}/mapping-source-field.html[`ctx['_source']`]::
+{ref}/mapping-source-field.html[`ctx['_source']`]::
         Modify the values in the `Map/List` structure to add, modify, or delete
         the fields of a document.
 
diff --git a/docs/painless/painless-contexts/painless-update-context.asciidoc b/docs/painless/painless-contexts/painless-update-context.asciidoc
index b04ba8d9ffb..d1b4b84eb18 100644
--- a/docs/painless/painless-contexts/painless-update-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-update-context.asciidoc
@@ -1,7 +1,7 @@
 [[painless-update-context]]
 === Update context
 
-Use a Painless script in an {es_version}/docs-update.html[update] operation to
+Use a Painless script in an {ref}/docs-update.html[update] operation to
 add, modify, or delete fields within a single document.
 
 *Variables*
@@ -12,16 +12,16 @@ add, modify, or delete fields within a single document.
 `ctx['_op']` (`String`)::
         The name of the operation.
 
-{es_version}/mapping-routing-field.html[`ctx['_routing']`] (`String`, read-only)::
+{ref}/mapping-routing-field.html[`ctx['_routing']`] (`String`, read-only)::
         The value used to select a shard for document storage.
 
-{es_version}/mapping-index-field.html[`ctx['_index']`] (`String`, read-only)::
+{ref}/mapping-index-field.html[`ctx['_index']`] (`String`, read-only)::
         The name of the index.
 
-{es_version}/mapping-type-field.html[`ctx['_type']`] (`String`, read-only)::
+{ref}/mapping-type-field.html[`ctx['_type']`] (`String`, read-only)::
         The type of document within an index.
 
-{es_version}/mapping-id-field.html[`ctx['_id']`] (`int`, read-only)::
+{ref}/mapping-id-field.html[`ctx['_id']`] (`int`, read-only)::
         The unique document id.
 
 `ctx['_version']` (`int`, read-only)::
@@ -30,7 +30,7 @@ add, modify, or delete fields within a single document.
 `ctx['_now']` (`long`, read-only)::
         The current timestamp in milliseconds.
 
-{es_version}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
+{ref}/mapping-source-field.html[`ctx['_source']`] (`Map`)::
         Contains extracted JSON in a `Map` and `List` structure for the fields
         existing in a stored document.
 
@@ -41,7 +41,7 @@ add, modify, or delete fields within a single document.
         specify no operation or `delete` to delete the current document from
         the index.
 
-{es_version}/mapping-source-field.html[`ctx['_source']`]::
+{ref}/mapping-source-field.html[`ctx['_source']`]::
         Modify the values in the `Map/List` structure to add, modify, or delete
         the fields of a document.
 
diff --git a/docs/painless/painless-contexts/painless-watcher-condition-context.asciidoc b/docs/painless/painless-contexts/painless-watcher-condition-context.asciidoc
index 3a5e460a55d..26efe24ae07 100644
--- a/docs/painless/painless-contexts/painless-watcher-condition-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-watcher-condition-context.asciidoc
@@ -1,7 +1,7 @@
 [[painless-watcher-condition-context]]
 === Watcher condition context
 
-Use a Painless script as a {xp_version}/condition-script.html[watcher condition]
+Use a Painless script as a {xpack-ref}/condition-script.html[watcher condition]
 to test if a response is necessary.
 
 *Variables*
@@ -26,7 +26,7 @@ to test if a response is necessary.
 
 `ctx['payload']` (`Map`, read-only)::
         The accessible watch data based upon the
-        {xp_version}/input.html[watch input].
+        {xpack-ref}/input.html[watch input].
 
 *Return*
 
diff --git a/docs/painless/painless-contexts/painless-watcher-transform-context.asciidoc b/docs/painless/painless-contexts/painless-watcher-transform-context.asciidoc
index 1831da5a9f8..9296fdda1c1 100644
--- a/docs/painless/painless-contexts/painless-watcher-transform-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-watcher-transform-context.asciidoc
@@ -1,7 +1,7 @@
 [[painless-watcher-transform-context]]
 === Watcher transform context
 
-Use a Painless script to {xp_version}/transform-script.html[transform] watch
+Use a Painless script to {xpack-ref}/transform-script.html[transform] watch
 data into a new payload for use in a response to a condition.
 
 *Variables*
@@ -26,7 +26,7 @@ data into a new payload for use in a response to a condition.
 
 `ctx['payload']` (`Map`, read-only)::
         The accessible watch data based upon the
-        {xp_version}/input.html[watch input].
+        {xpack-ref}/input.html[watch input].
 
 
 *Return*
diff --git a/docs/painless/painless-contexts/painless-weight-context.asciidoc b/docs/painless/painless-contexts/painless-weight-context.asciidoc
index 0aef936183c..ad215d5386b 100644
--- a/docs/painless/painless-contexts/painless-weight-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-weight-context.asciidoc
@@ -2,7 +2,7 @@
 === Weight context
 
 Use a Painless script to create a
-{es_version}/index-modules-similarity.html[weight] for use in a
+{ref}/index-modules-similarity.html[weight] for use in a
 <<painless-similarity-context, similarity script>>.  Weight is used to prevent
 recalculation of constants that remain the same across documents.
 

From a577fb3381c8c9229502677e8d403f5fdd7c0704 Mon Sep 17 00:00:00 2001
From: lcawl <lcawley@elastic.co>
Date: Wed, 25 Jul 2018 14:19:19 -0700
Subject: [PATCH 185/260] [DOCS] Fix formatting error in Slack action

---
 x-pack/docs/en/watcher/actions/slack.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/docs/en/watcher/actions/slack.asciidoc b/x-pack/docs/en/watcher/actions/slack.asciidoc
index 0753f333dc7..0d0f77a4a30 100644
--- a/x-pack/docs/en/watcher/actions/slack.asciidoc
+++ b/x-pack/docs/en/watcher/actions/slack.asciidoc
@@ -169,9 +169,9 @@ aggregation and the Slack action:
                                     more information, see
                                     <<slack-dynamic-attachment, Using Attachments to Format Slack Messages>>.
 
-| `proxy.host`      | no        | -       | The proxy host to use (only in combination with `proxy.port`)
+| `proxy.host`      | no        | The proxy host to use (only in combination with `proxy.port`)
 
-| `proxy.port`      | no        | -       | The proxy port to use (only in combination with `proxy.host`)
+| `proxy.port`      | no        | The proxy port to use (only in combination with `proxy.host`)
 |======
 
 [[configuring-slack]]

From 467a60ba0c7b5cde5275d9a18042a3df5a40b1ab Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 25 Jul 2018 20:57:03 -0400
Subject: [PATCH 186/260] Only enforce password hashing check if FIPS enabled
 (#32383)

This commit modifies the FIPS password hashing algorithm check to only
be executed if FIPS mode is enabled.
---
 ...asswordHashingAlgorithmBootstrapCheck.java | 18 +++---
 ...rdHashingAlgorithmBootstrapCheckTests.java | 56 ++++++++++++++-----
 2 files changed, 50 insertions(+), 24 deletions(-)

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
index 7f6d799cf5a..57e12a211a3 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
@@ -16,7 +16,7 @@ public class FIPS140PasswordHashingAlgorithmBootstrapCheck implements BootstrapC
 
     private final boolean fipsModeEnabled;
 
-    FIPS140PasswordHashingAlgorithmBootstrapCheck(Settings settings) {
+    FIPS140PasswordHashingAlgorithmBootstrapCheck(final Settings settings) {
         this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
     }
 
@@ -27,17 +27,15 @@ public class FIPS140PasswordHashingAlgorithmBootstrapCheck implements BootstrapC
      * @return the result of the bootstrap check
      */
     @Override
-    public BootstrapCheckResult check(BootstrapContext context) {
-        final String selectedAlgorithm = XPackSettings.PASSWORD_HASHING_ALGORITHM.get(context.settings);
-        if (selectedAlgorithm.toLowerCase(Locale.ROOT).startsWith("pbkdf2") == false) {
-            return BootstrapCheckResult.failure("Only PBKDF2 is allowed for password hashing in a FIPS-140 JVM. Please set the " +
-                "appropriate value for [ " + XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey() + " ] setting.");
+    public BootstrapCheckResult check(final BootstrapContext context) {
+        if (fipsModeEnabled) {
+            final String selectedAlgorithm = XPackSettings.PASSWORD_HASHING_ALGORITHM.get(context.settings);
+            if (selectedAlgorithm.toLowerCase(Locale.ROOT).startsWith("pbkdf2") == false) {
+                return BootstrapCheckResult.failure("Only PBKDF2 is allowed for password hashing in a FIPS-140 JVM. Please set the " +
+                        "appropriate value for [ " + XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey() + " ] setting.");
+            }
         }
         return BootstrapCheckResult.success();
     }
 
-    @Override
-    public boolean alwaysEnforce() {
-        return fipsModeEnabled;
-    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
index 310a6e241e0..b424986876a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
@@ -3,32 +3,60 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
 package org.elasticsearch.xpack.security;
 
+import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.bootstrap.BootstrapContext;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.XPackSettings;
 
+import java.util.Arrays;
+
+import static org.hamcrest.Matchers.equalTo;
+
 public class FIPS140PasswordHashingAlgorithmBootstrapCheckTests extends ESTestCase {
 
     public void testPBKDF2AlgorithmIsAllowed() {
-        Settings settings = Settings.builder().put("xpack.security.fips_mode.enabled", "true").build();
+        {
+            final Settings settings = Settings.builder()
+                    .put(Security.FIPS_MODE_ENABLED.getKey(), true)
+                    .put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2_10000")
+                    .build();
+            final BootstrapCheck.BootstrapCheckResult result =
+                    new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null));
+            assertFalse(result.isFailure());
+        }
 
-        settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2_10000").build();
-        assertFalse(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
-
-        settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2").build();
-        assertFalse(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
+        {
+            final Settings settings = Settings.builder()
+                    .put(Security.FIPS_MODE_ENABLED.getKey(), true)
+                    .put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2")
+                    .build();
+            final BootstrapCheck.BootstrapCheckResult result =
+                    new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null));
+            assertFalse(result.isFailure());
+        }
     }
 
-    public void testBCRYPTAlgorithmIsNotAllowed() {
-        Settings settings = Settings.builder().put("xpack.security.fips_mode.enabled", "true").build();
-        assertTrue(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
-        settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "BCRYPT").build();
-        assertTrue(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
-
-        settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "BCRYPT11").build();
-        assertTrue(new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null)).isFailure());
+    public void testBCRYPTAlgorithmDependsOnFipsMode() {
+        for (final Boolean fipsModeEnabled : Arrays.asList(true, false)) {
+            for (final String passwordHashingAlgorithm : Arrays.asList(null, "BCRYPT", "BCRYPT11")) {
+                runBCRYPTTest(fipsModeEnabled, passwordHashingAlgorithm);
+            }
+        }
     }
+
+    private void runBCRYPTTest(final boolean fipsModeEnabled, final String passwordHashingAlgorithm) {
+        final Settings.Builder builder = Settings.builder().put(Security.FIPS_MODE_ENABLED.getKey(), fipsModeEnabled);
+        if (passwordHashingAlgorithm != null) {
+            builder.put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), passwordHashingAlgorithm);
+        }
+        final Settings settings = builder.build();
+        final BootstrapCheck.BootstrapCheckResult result =
+                new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings).check(new BootstrapContext(settings, null));
+        assertThat(result.isFailure(), equalTo(fipsModeEnabled));
+    }
+
 }

From 48885d2b7e0fd675ab6db791c9c3e021efcb2f2d Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Thu, 26 Jul 2018 07:01:59 +0200
Subject: [PATCH 187/260] TESTS: Fix Buf Leaks in HttpReadWriteHandlerTests
 (#32377)

* TESTS: Fix Buf Leaks in HttpReadWriteHandlerTests

* Release all ref counted things that weren't getting properly released
* Mannually force channel promise to be completed because mock channel doesn't do it and it prevents one `release` call in `io.netty.channel.ChannelOutboundHandlerAdapter#write` from firing
---
 .../http/nio/HttpReadWriteHandlerTests.java   | 192 +++++++++++-------
 1 file changed, 124 insertions(+), 68 deletions(-)

diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
index 0a09b6b8789..62bf845a770 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.http.nio;
 
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelPromise;
 import io.netty.channel.embedded.EmbeddedChannel;
 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
@@ -116,21 +117,27 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
 
         ByteBuf buf = requestEncoder.encode(httpRequest);
         int slicePoint = randomInt(buf.writerIndex() - 1);
-
         ByteBuf slicedBuf = buf.retainedSlice(0, slicePoint);
         ByteBuf slicedBuf2 = buf.retainedSlice(slicePoint, buf.writerIndex());
-        handler.consumeReads(toChannelBuffer(slicedBuf));
+        try {
+            handler.consumeReads(toChannelBuffer(slicedBuf));
 
-        verify(transport, times(0)).incomingRequest(any(HttpRequest.class), any(NioHttpChannel.class));
+            verify(transport, times(0)).incomingRequest(any(HttpRequest.class), any(NioHttpChannel.class));
 
-        handler.consumeReads(toChannelBuffer(slicedBuf2));
+            handler.consumeReads(toChannelBuffer(slicedBuf2));
 
-        ArgumentCaptor<HttpRequest> requestCaptor = ArgumentCaptor.forClass(HttpRequest.class);
-        verify(transport).incomingRequest(requestCaptor.capture(), any(NioHttpChannel.class));
+            ArgumentCaptor<HttpRequest> requestCaptor = ArgumentCaptor.forClass(HttpRequest.class);
+            verify(transport).incomingRequest(requestCaptor.capture(), any(NioHttpChannel.class));
 
-        HttpRequest nioHttpRequest = requestCaptor.getValue();
-        assertEquals(HttpRequest.HttpVersion.HTTP_1_1, nioHttpRequest.protocolVersion());
-        assertEquals(RestRequest.Method.GET, nioHttpRequest.method());
+            HttpRequest nioHttpRequest = requestCaptor.getValue();
+            assertEquals(HttpRequest.HttpVersion.HTTP_1_1, nioHttpRequest.protocolVersion());
+            assertEquals(RestRequest.Method.GET, nioHttpRequest.method());
+        } finally {
+            handler.close();
+            buf.release();
+            slicedBuf.release();
+            slicedBuf2.release();
+        }
     }
 
     public void testDecodeHttpRequestError() throws IOException {
@@ -138,16 +145,20 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
         io.netty.handler.codec.http.HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
 
         ByteBuf buf = requestEncoder.encode(httpRequest);
-        buf.setByte(0, ' ');
-        buf.setByte(1, ' ');
-        buf.setByte(2, ' ');
+        try {
+            buf.setByte(0, ' ');
+            buf.setByte(1, ' ');
+            buf.setByte(2, ' ');
 
-        handler.consumeReads(toChannelBuffer(buf));
+            handler.consumeReads(toChannelBuffer(buf));
 
-        ArgumentCaptor<Exception> exceptionCaptor = ArgumentCaptor.forClass(Exception.class);
-        verify(transport).incomingRequestError(any(HttpRequest.class), any(NioHttpChannel.class), exceptionCaptor.capture());
+            ArgumentCaptor<Exception> exceptionCaptor = ArgumentCaptor.forClass(Exception.class);
+            verify(transport).incomingRequestError(any(HttpRequest.class), any(NioHttpChannel.class), exceptionCaptor.capture());
 
-        assertTrue(exceptionCaptor.getValue() instanceof IllegalArgumentException);
+            assertTrue(exceptionCaptor.getValue() instanceof IllegalArgumentException);
+        } finally {
+            buf.release();
+        }
     }
 
     public void testDecodeHttpRequestContentLengthToLongGeneratesOutboundMessage() throws IOException {
@@ -157,9 +168,11 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
         HttpUtil.setKeepAlive(httpRequest, false);
 
         ByteBuf buf = requestEncoder.encode(httpRequest);
-
-        handler.consumeReads(toChannelBuffer(buf));
-
+        try {
+            handler.consumeReads(toChannelBuffer(buf));
+        } finally {
+            buf.release();
+        }
         verify(transport, times(0)).incomingRequestError(any(), any(), any());
         verify(transport, times(0)).incomingRequest(any(), any());
 
@@ -168,13 +181,17 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
 
         FlushOperation flushOperation = flushOperations.get(0);
         FullHttpResponse response = responseDecoder.decode(Unpooled.wrappedBuffer(flushOperation.getBuffersToWrite()));
-        assertEquals(HttpVersion.HTTP_1_1, response.protocolVersion());
-        assertEquals(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, response.status());
+        try {
+            assertEquals(HttpVersion.HTTP_1_1, response.protocolVersion());
+            assertEquals(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, response.status());
 
-        flushOperation.getListener().accept(null, null);
-        // Since we have keep-alive set to false, we should close the channel after the response has been
-        // flushed
-        verify(nioHttpChannel).close();
+            flushOperation.getListener().accept(null, null);
+            // Since we have keep-alive set to false, we should close the channel after the response has been
+            // flushed
+            verify(nioHttpChannel).close();
+        } finally {
+            response.release();
+        }
     }
 
     @SuppressWarnings("unchecked")
@@ -189,11 +206,15 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
         SocketChannelContext context = mock(SocketChannelContext.class);
         HttpWriteOperation writeOperation = new HttpWriteOperation(context, httpResponse, mock(BiConsumer.class));
         List<FlushOperation> flushOperations = handler.writeToBytes(writeOperation);
-
-        FullHttpResponse response = responseDecoder.decode(Unpooled.wrappedBuffer(flushOperations.get(0).getBuffersToWrite()));
-
-        assertEquals(HttpResponseStatus.OK, response.status());
-        assertEquals(HttpVersion.HTTP_1_1, response.protocolVersion());
+        FlushOperation operation = flushOperations.get(0);
+        FullHttpResponse response = responseDecoder.decode(Unpooled.wrappedBuffer(operation.getBuffersToWrite()));
+        ((ChannelPromise) operation.getListener()).setSuccess();
+        try {
+            assertEquals(HttpResponseStatus.OK, response.status());
+            assertEquals(HttpVersion.HTTP_1_1, response.protocolVersion());
+        } finally {
+            response.release();
+        }
     }
 
     public void testCorsEnabledWithoutAllowOrigins() throws IOException {
@@ -201,9 +222,13 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
         Settings settings = Settings.builder()
             .put(HttpTransportSettings.SETTING_CORS_ENABLED.getKey(), true)
             .build();
-        io.netty.handler.codec.http.HttpResponse response = executeCorsRequest(settings, "remote-host", "request-host");
-        // inspect response and validate
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), nullValue());
+        FullHttpResponse response = executeCorsRequest(settings, "remote-host", "request-host");
+        try {
+            // inspect response and validate
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), nullValue());
+        } finally {
+            response.release();
+        }
     }
 
     public void testCorsEnabledWithAllowOrigins() throws IOException {
@@ -213,11 +238,15 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
             .put(SETTING_CORS_ENABLED.getKey(), true)
             .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
             .build();
-        io.netty.handler.codec.http.HttpResponse response = executeCorsRequest(settings, originValue, "request-host");
-        // inspect response and validate
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
-        String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
-        assertThat(allowedOrigins, is(originValue));
+        FullHttpResponse response = executeCorsRequest(settings, originValue, "request-host");
+        try {
+            // inspect response and validate
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+            String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
+            assertThat(allowedOrigins, is(originValue));
+        } finally {
+            response.release();
+        }
     }
 
     public void testCorsAllowOriginWithSameHost() throws IOException {
@@ -228,29 +257,44 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
             .put(SETTING_CORS_ENABLED.getKey(), true)
             .build();
         FullHttpResponse response = executeCorsRequest(settings, originValue, host);
-        // inspect response and validate
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
-        String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
-        assertThat(allowedOrigins, is(originValue));
-
+        String allowedOrigins;
+        try {
+            // inspect response and validate
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+            allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
+            assertThat(allowedOrigins, is(originValue));
+        } finally {
+            response.release();
+        }
         originValue = "http://" + originValue;
         response = executeCorsRequest(settings, originValue, host);
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
-        allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
-        assertThat(allowedOrigins, is(originValue));
+        try {
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+            allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
+            assertThat(allowedOrigins, is(originValue));
+        } finally {
+            response.release();
+        }
 
         originValue = originValue + ":5555";
         host = host + ":5555";
         response = executeCorsRequest(settings, originValue, host);
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
-        allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
-        assertThat(allowedOrigins, is(originValue));
-
+        try {
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+            allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
+            assertThat(allowedOrigins, is(originValue));
+        } finally {
+            response.release();
+        }
         originValue = originValue.replace("http", "https");
         response = executeCorsRequest(settings, originValue, host);
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
-        allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
-        assertThat(allowedOrigins, is(originValue));
+        try {
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+            allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
+            assertThat(allowedOrigins, is(originValue));
+        } finally {
+            response.release();
+        }
     }
 
     public void testThatStringLiteralWorksOnMatch() throws IOException {
@@ -261,12 +305,16 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
             .put(SETTING_CORS_ALLOW_METHODS.getKey(), "get, options, post")
             .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
             .build();
-        io.netty.handler.codec.http.HttpResponse response = executeCorsRequest(settings, originValue, "request-host");
-        // inspect response and validate
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
-        String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
-        assertThat(allowedOrigins, is(originValue));
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), equalTo("true"));
+        FullHttpResponse response = executeCorsRequest(settings, originValue, "request-host");
+        try {
+            // inspect response and validate
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+            String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
+            assertThat(allowedOrigins, is(originValue));
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), equalTo("true"));
+        } finally {
+            response.release();
+        }
     }
 
     public void testThatAnyOriginWorks() throws IOException {
@@ -275,12 +323,16 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
             .put(SETTING_CORS_ENABLED.getKey(), true)
             .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
             .build();
-        io.netty.handler.codec.http.HttpResponse response = executeCorsRequest(settings, originValue, "request-host");
-        // inspect response and validate
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
-        String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
-        assertThat(allowedOrigins, is(originValue));
-        assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), nullValue());
+        FullHttpResponse response = executeCorsRequest(settings, originValue, "request-host");
+        try {
+            // inspect response and validate
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+            String allowedOrigins = response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN);
+            assertThat(allowedOrigins, is(originValue));
+            assertThat(response.headers().get(HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS), nullValue());
+        } finally {
+            response.release();
+        }
     }
 
     private FullHttpResponse executeCorsRequest(final Settings settings, final String originValue, final String host) throws IOException {
@@ -300,8 +352,9 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
 
         SocketChannelContext context = mock(SocketChannelContext.class);
         List<FlushOperation> flushOperations = handler.writeToBytes(handler.createWriteOperation(context, response, (v, e) -> {}));
-
+        handler.close();
         FlushOperation flushOperation = flushOperations.get(0);
+        ((ChannelPromise) flushOperation.getListener()).setSuccess();
         return responseDecoder.decode(Unpooled.wrappedBuffer(flushOperation.getBuffersToWrite()));
     }
 
@@ -314,8 +367,11 @@ public class HttpReadWriteHandlerTests extends ESTestCase {
 
         io.netty.handler.codec.http.HttpRequest request = new DefaultFullHttpRequest(version, method, uri);
         ByteBuf buf = requestEncoder.encode(request);
-
-        handler.consumeReads(toChannelBuffer(buf));
+        try {
+            handler.consumeReads(toChannelBuffer(buf));
+        } finally {
+            buf.release();
+        }
 
         ArgumentCaptor<NioHttpRequest> requestCaptor = ArgumentCaptor.forClass(NioHttpRequest.class);
         verify(transport, atLeastOnce()).incomingRequest(requestCaptor.capture(), any(HttpChannel.class));

From f7ba14d5f478aca194e8f56bca9fd63cf6010996 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 26 Jul 2018 09:02:58 +0200
Subject: [PATCH 188/260] Make sure _forcemerge respects `max_num_segments`.
 (#32291)

An upcoming [Lucene change](https://issues.apache.org/jira/browse/LUCENE-7976)
will make TieredMergePolicy respect the maximum merged segment size all the
time, meaning it will possibly not respect the `max_num_segments` parameter
anymore if the shard is larger than the maximum segment size.

This change makes sure that `max_num_segments` is respected for now in order
to give us time to think about how to integrate this change, and also to delay
it until 7.0 as this might be a big-enough change for us to wait for a new
major version.
---
 .../index/EsTieredMergePolicy.java            | 120 ++++++++++++++++++
 .../index/MergePolicyConfig.java              |   2 +-
 .../index/EsTieredMergePolicyTests.java       |  80 ++++++++++++
 .../index/MergePolicySettingsTests.java       |  43 +++----
 4 files changed, 222 insertions(+), 23 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java

diff --git a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java
new file mode 100644
index 00000000000..27a83961903
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index;
+
+import org.apache.lucene.index.FilterMergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.TieredMergePolicy;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Wrapper around {@link TieredMergePolicy} which doesn't respect
+ * {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges.
+ * See https://issues.apache.org/jira/browse/LUCENE-7976.
+ */
+final class EsTieredMergePolicy extends FilterMergePolicy {
+
+    final TieredMergePolicy regularMergePolicy;
+    final TieredMergePolicy forcedMergePolicy;
+
+    EsTieredMergePolicy() {
+        super(new TieredMergePolicy());
+        regularMergePolicy = (TieredMergePolicy) in;
+        forcedMergePolicy = new TieredMergePolicy();
+        forcedMergePolicy.setMaxMergedSegmentMB(Double.POSITIVE_INFINITY); // unlimited
+    }
+
+    @Override
+    public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount,
+            Map<SegmentCommitInfo, Boolean> segmentsToMerge, MergeContext mergeContext) throws IOException {
+        return forcedMergePolicy.findForcedMerges(infos, maxSegmentCount, segmentsToMerge, mergeContext);
+    }
+
+    @Override
+    public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
+        return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);    
+    }
+
+    public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
+        regularMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
+        forcedMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
+    }
+
+    public double getForceMergeDeletesPctAllowed() {
+        return forcedMergePolicy.getForceMergeDeletesPctAllowed();
+    }
+
+    public void setFloorSegmentMB(double mbFrac) {
+        regularMergePolicy.setFloorSegmentMB(mbFrac);
+        forcedMergePolicy.setFloorSegmentMB(mbFrac);
+    }
+
+    public double getFloorSegmentMB() {
+        return regularMergePolicy.getFloorSegmentMB();
+    }
+
+    public void setMaxMergeAtOnce(int maxMergeAtOnce) {
+        regularMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
+        forcedMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
+    }
+
+    public int getMaxMergeAtOnce() {
+        return regularMergePolicy.getMaxMergeAtOnce();
+    }
+
+    public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) {
+        regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
+        forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
+    }
+
+    public int getMaxMergeAtOnceExplicit() {
+        return forcedMergePolicy.getMaxMergeAtOnceExplicit();
+    }
+
+    // only setter that must NOT delegate to the forced merge policy
+    public void setMaxMergedSegmentMB(double mbFrac) {
+        regularMergePolicy.setMaxMergedSegmentMB(mbFrac);
+    }
+
+    public double getMaxMergedSegmentMB() {
+        return regularMergePolicy.getMaxMergedSegmentMB();
+    }
+
+    public void setSegmentsPerTier(double segmentsPerTier) {
+        regularMergePolicy.setSegmentsPerTier(segmentsPerTier);
+        forcedMergePolicy.setSegmentsPerTier(segmentsPerTier);
+    }
+
+    public double getSegmentsPerTier() {
+        return regularMergePolicy.getSegmentsPerTier();
+    }
+
+    public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
+        regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
+        forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
+    }
+
+    public double getReclaimDeletesWeight() {
+        return regularMergePolicy.getReclaimDeletesWeight();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
index 0f7305789ec..b6c1e124a63 100644
--- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
+++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
@@ -115,7 +115,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
  */
 
 public final class MergePolicyConfig {
-    private final TieredMergePolicy mergePolicy = new TieredMergePolicy();
+    private final EsTieredMergePolicy mergePolicy = new EsTieredMergePolicy();
     private final Logger logger;
     private final boolean mergesEnabled;
 
diff --git a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java
new file mode 100644
index 00000000000..fdee707d97a
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index;
+
+import org.apache.lucene.index.TieredMergePolicy;
+import org.elasticsearch.test.ESTestCase;
+
+public class EsTieredMergePolicyTests extends ESTestCase {
+
+    public void testDefaults() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        assertEquals(
+                new TieredMergePolicy().getMaxMergedSegmentMB(),
+                policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
+        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
+        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+    }
+
+    public void testSetMaxMergedSegmentMB() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setMaxMergedSegmentMB(10 * 1024);
+        assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
+        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
+        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+    }
+
+    public void testSetForceMergeDeletesPctAllowed() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setForceMergeDeletesPctAllowed(42);
+        assertEquals(42, policy.forcedMergePolicy.getForceMergeDeletesPctAllowed(), 0);
+    }
+
+    public void testSetFloorSegmentMB() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setFloorSegmentMB(42);
+        assertEquals(42, policy.regularMergePolicy.getFloorSegmentMB(), 0);
+        assertEquals(42, policy.forcedMergePolicy.getFloorSegmentMB(), 0);
+    }
+
+    public void testSetMaxMergeAtOnce() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setMaxMergeAtOnce(42);
+        assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce());
+    }
+
+    public void testSetMaxMergeAtOnceExplicit() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setMaxMergeAtOnceExplicit(42);
+        assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit());
+    }
+
+    public void testSetSegmentsPerTier() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setSegmentsPerTier(42);
+        assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
+    }
+
+    public void testSetReclaimDeletesWeight() {
+        EsTieredMergePolicy policy = new EsTieredMergePolicy();
+        policy.setReclaimDeletesWeight(42);
+        assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
index ff60a206229..d8b03ffeedb 100644
--- a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.index;
 
 import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.index.TieredMergePolicy;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -76,43 +75,43 @@ public class MergePolicySettingsTests extends ESTestCase {
 
     public void testTieredMergePolicySettingsUpdate() throws IOException {
         IndexSettings indexSettings = indexSettings(Settings.EMPTY);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
 
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
 
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
 
         indexSettings.updateIndexMetaData(newIndexMeta("index", EMPTY_SETTINGS)); // see if defaults are restored
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
+        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
     }
 
     public Settings build(String value) {

From bec888fa78e67d2221af2962bbca8e6ed0d9cbfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Thu, 26 Jul 2018 11:15:53 +0200
Subject: [PATCH 189/260] Rank-Eval: Reduce scope of an unchecked supression

We should only supress the unchecked warnings on ConstructingObjectParser.
---
 .../java/org/elasticsearch/index/rankeval/RatedRequest.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java
index 79dd693b3ac..2fe3f1922d5 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java
@@ -70,7 +70,6 @@ import java.util.Set;
  * }
  * </pre>
  */
-@SuppressWarnings("unchecked")
 public class RatedRequest implements Writeable, ToXContentObject {
     private final String id;
     private final List<String> summaryFields;
@@ -250,6 +249,7 @@ public class RatedRequest implements Writeable, ToXContentObject {
     private static final ParseField FIELDS_FIELD = new ParseField("summary_fields");
     private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id");
 
+    @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<RatedRequest, Void> PARSER = new ConstructingObjectParser<>("request",
             a -> new RatedRequest((String) a[0], (List<RatedDocument>) a[1], (SearchSourceBuilder) a[2], (Map<String, Object>) a[3],
                     (String) a[4]));

From eb675a1c4d2660c7f0090595d0771809245030da Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 26 Jul 2018 08:05:49 -0400
Subject: [PATCH 190/260] Introduce index store plugins (#32375)

Today we allow plugins to add index store implementations yet we are not
doing this in our new way of managing plugins as pull versus push. That
is, today we still allow plugins to push index store providers via an on
index module call where they can turn around and add an index
store. Aside from being inconsistent with how we manage plugins today
where we would look to pull such implementations from plugins at node
creation time, it also means that we do not know at a top-level (for
example, in the indices service) which index stores are available. This
commit addresses this by adding a dedicated plugin type for index store
plugins, removing the index module hook for adding index stores, and by
aggregating these into the top-level of the indices service.
---
 .../plugin/store/smb/SMBStorePlugin.java      | 20 ++++--
 .../org/elasticsearch/index/IndexModule.java  | 40 ++++-------
 .../elasticsearch/indices/IndicesService.java | 10 ++-
 .../java/org/elasticsearch/node/Node.java     | 14 +++-
 .../plugins/IndexStorePlugin.java             | 42 +++++++++++
 .../elasticsearch/index/IndexModuleTests.java | 61 ++++++++--------
 .../plugins/IndexStorePluginTests.java        | 72 +++++++++++++++++++
 .../test/store/MockFSIndexStore.java          | 10 ++-
 .../xpack/watcher/WatcherPluginTests.java     |  3 +-
 9 files changed, 202 insertions(+), 70 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java
 create mode 100644 server/src/test/java/org/elasticsearch/plugins/IndexStorePluginTests.java

diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java
index 241f3d77e8b..a2b0f463452 100644
--- a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java
+++ b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java
@@ -19,16 +19,26 @@
 
 package org.elasticsearch.plugin.store.smb;
 
-import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.smbmmapfs.SmbMmapFsIndexStore;
 import org.elasticsearch.index.store.smbsimplefs.SmbSimpleFsIndexStore;
+import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class SMBStorePlugin extends Plugin {
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+public class SMBStorePlugin extends Plugin implements IndexStorePlugin {
 
     @Override
-    public void onIndexModule(IndexModule indexModule) {
-        indexModule.addIndexStore("smb_mmap_fs", SmbMmapFsIndexStore::new);
-        indexModule.addIndexStore("smb_simple_fs", SmbSimpleFsIndexStore::new);
+    public Map<String, Function<IndexSettings, IndexStore>> getIndexStoreFactories() {
+        final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories = new HashMap<>(2);
+        indexStoreFactories.put("smb_mmap_fs", SmbMmapFsIndexStore::new);
+        indexStoreFactories.put("smb_simple_fs", SmbSimpleFsIndexStore::new);
+        return Collections.unmodifiableMap(indexStoreFactories);
     }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java
index 9e859a16956..715b78b14ff 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -49,6 +49,7 @@ import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -74,7 +75,7 @@ import java.util.function.Function;
  *     {@link #addSimilarity(String, TriFunction)} while existing Providers can be referenced through Settings under the
  *     {@link IndexModule#SIMILARITY_SETTINGS_PREFIX} prefix along with the "type" value.  For example, to reference the
  *     {@link BM25Similarity}, the configuration {@code "index.similarity.my_similarity.type : "BM25"} can be used.</li>
- *      <li>{@link IndexStore} - Custom {@link IndexStore} instances can be registered via {@link #addIndexStore(String, Function)}</li>
+ *      <li>{@link IndexStore} - Custom {@link IndexStore} instances can be registered via {@link IndexStorePlugin}</li>
  *      <li>{@link IndexEventListener} - Custom {@link IndexEventListener} instances can be registered via
  *      {@link #addIndexEventListener(IndexEventListener)}</li>
  *      <li>Settings update listener - Custom settings update listener can be registered via
@@ -109,7 +110,7 @@ public final class IndexModule {
     private SetOnce<IndexSearcherWrapperFactory> indexSearcherWrapper = new SetOnce<>();
     private final Set<IndexEventListener> indexEventListeners = new HashSet<>();
     private final Map<String, TriFunction<Settings, Version, ScriptService, Similarity>> similarities = new HashMap<>();
-    private final Map<String, Function<IndexSettings, IndexStore>> storeTypes = new HashMap<>();
+    private final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories;
     private final SetOnce<BiFunction<IndexSettings, IndicesQueryCache, QueryCache>> forceQueryCacheProvider = new SetOnce<>();
     private final List<SearchOperationListener> searchOperationListeners = new ArrayList<>();
     private final List<IndexingOperationListener> indexOperationListeners = new ArrayList<>();
@@ -119,16 +120,22 @@ public final class IndexModule {
      * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins
      * via {@link org.elasticsearch.plugins.PluginsService#onIndexModule(IndexModule)}.
      *
-     * @param indexSettings    the index settings
-     * @param analysisRegistry the analysis registry
-     * @param engineFactory    the engine factory
+     * @param indexSettings       the index settings
+     * @param analysisRegistry    the analysis registry
+     * @param engineFactory       the engine factory
+     * @param indexStoreFactories the available store types
      */
-    public IndexModule(final IndexSettings indexSettings, final AnalysisRegistry analysisRegistry, final EngineFactory engineFactory) {
+    public IndexModule(
+            final IndexSettings indexSettings,
+            final AnalysisRegistry analysisRegistry,
+            final EngineFactory engineFactory,
+            final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories) {
         this.indexSettings = indexSettings;
         this.analysisRegistry = analysisRegistry;
         this.engineFactory = Objects.requireNonNull(engineFactory);
         this.searchOperationListeners.add(new SearchSlowLog(indexSettings));
         this.indexOperationListeners.add(new IndexingSlowLog(indexSettings));
+        this.indexStoreFactories = Collections.unmodifiableMap(indexStoreFactories);
     }
 
     /**
@@ -245,25 +252,6 @@ public final class IndexModule {
         this.indexOperationListeners.add(listener);
     }
 
-    /**
-     * Adds an {@link IndexStore} type to this index module. Typically stores are registered with a reference to
-     * it's constructor:
-     * <pre>
-     *     indexModule.addIndexStore("my_store_type", MyStore::new);
-     * </pre>
-     *
-     * @param type the type to register
-     * @param provider the instance provider / factory method
-     */
-    public void addIndexStore(String type, Function<IndexSettings, IndexStore> provider) {
-        ensureNotFrozen();
-        if (storeTypes.containsKey(type)) {
-            throw new IllegalArgumentException("key [" + type +"] already registered");
-        }
-        storeTypes.put(type, provider);
-    }
-
-
     /**
      * Registers the given {@link Similarity} with the given name.
      * The function takes as parameters:<ul>
@@ -360,7 +348,7 @@ public final class IndexModule {
         if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) {
             store = new IndexStore(indexSettings);
         } else {
-            Function<IndexSettings, IndexStore> factory = storeTypes.get(storeType);
+            Function<IndexSettings, IndexStore> factory = indexStoreFactories.get(storeType);
             if (factory == null) {
                 throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
             }
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
index 4f535f01da4..39346fecbef 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -100,6 +100,7 @@ import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
@@ -181,6 +182,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private final IndicesQueryCache indicesQueryCache;
     private final MetaStateService metaStateService;
     private final Collection<Function<IndexSettings, Optional<EngineFactory>>> engineFactoryProviders;
+    private final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories;
 
     @Override
     protected void doStart() {
@@ -193,7 +195,8 @@ public class IndicesService extends AbstractLifecycleComponent
                           MapperRegistry mapperRegistry, NamedWriteableRegistry namedWriteableRegistry, ThreadPool threadPool,
                           IndexScopedSettings indexScopedSettings, CircuitBreakerService circuitBreakerService, BigArrays bigArrays,
                           ScriptService scriptService, Client client, MetaStateService metaStateService,
-                          Collection<Function<IndexSettings, Optional<EngineFactory>>> engineFactoryProviders) {
+                          Collection<Function<IndexSettings, Optional<EngineFactory>>> engineFactoryProviders,
+                          Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories) {
         super(settings);
         this.threadPool = threadPool;
         this.pluginsService = pluginsService;
@@ -225,6 +228,7 @@ public class IndicesService extends AbstractLifecycleComponent
         this.cacheCleaner = new CacheCleaner(indicesFieldDataCache, indicesRequestCache,  logger, threadPool, this.cleanInterval);
         this.metaStateService = metaStateService;
         this.engineFactoryProviders = engineFactoryProviders;
+        this.indexStoreFactories = indexStoreFactories;
     }
 
     @Override
@@ -464,7 +468,7 @@ public class IndicesService extends AbstractLifecycleComponent
             idxSettings.getNumberOfReplicas(),
             reason);
 
-        final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry, getEngineFactory(idxSettings));
+        final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry, getEngineFactory(idxSettings), indexStoreFactories);
         for (IndexingOperationListener operationListener : indexingOperationListeners) {
             indexModule.addIndexOperationListener(operationListener);
         }
@@ -524,7 +528,7 @@ public class IndicesService extends AbstractLifecycleComponent
      */
     public synchronized MapperService createIndexMapperService(IndexMetaData indexMetaData) throws IOException {
         final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopedSettings);
-        final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry, getEngineFactory(idxSettings));
+        final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry, getEngineFactory(idxSettings), indexStoreFactories);
         pluginsService.onIndexModule(indexModule);
         return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService);
     }
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 64bc55edb71..c1ce864223b 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -26,8 +26,8 @@ import org.elasticsearch.Build;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.search.SearchExecutionStatsCollector;
 import org.elasticsearch.action.search.SearchPhaseController;
 import org.elasticsearch.action.search.SearchTransportService;
@@ -94,6 +94,7 @@ import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.engine.EngineFactory;
+import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
@@ -117,6 +118,7 @@ import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.EnginePlugin;
+import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.MetaDataUpgrader;
@@ -407,11 +409,19 @@ public class Node implements Closeable {
                             enginePlugins.stream().map(plugin -> plugin::getEngineFactory))
                     .collect(Collectors.toList());
 
+
+            final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories =
+                    pluginsService.filterPlugins(IndexStorePlugin.class)
+                            .stream()
+                            .map(IndexStorePlugin::getIndexStoreFactories)
+                            .flatMap(m -> m.entrySet().stream())
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
             final IndicesService indicesService =
                     new IndicesService(settings, pluginsService, nodeEnvironment, xContentRegistry, analysisModule.getAnalysisRegistry(),
                             clusterModule.getIndexNameExpressionResolver(), indicesModule.getMapperRegistry(), namedWriteableRegistry,
                             threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays,
-                            scriptModule.getScriptService(), client, metaStateService, engineFactoryProviders);
+                            scriptModule.getScriptService(), client, metaStateService, engineFactoryProviders, indexStoreFactories);
 
 
             Collection<Object> pluginComponents = pluginsService.filterPlugins(Plugin.class).stream()
diff --git a/server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java
new file mode 100644
index 00000000000..16eec535e4b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.store.IndexStore;
+
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * A plugin that provides alternative index store implementations.
+ */
+public interface IndexStorePlugin {
+
+    /**
+     * The index store factories for this plugin. When an index is created the store type setting
+     * {@link org.elasticsearch.index.IndexModule#INDEX_STORE_TYPE_SETTING} on the index will be examined and either use the default or a
+     * built-in type, or looked up among all the index store factories from {@link IndexStore} plugins.
+     *
+     * @return a map from store type to an index store factory
+     */
+    Map<String, Function<IndexSettings, IndexStore>> getIndexStoreFactories();
+
+}
diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 1d531bdeb90..a82b932e2b5 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -81,10 +81,13 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
 
 import static java.util.Collections.emptyMap;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class IndexModuleTests extends ESTestCase {
     private Index index;
@@ -147,7 +150,8 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     public void testWrapperIsBound() throws IOException {
-        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new MockEngineFactory(AssertingDirectoryReader.class));
+        final MockEngineFactory engineFactory = new MockEngineFactory(AssertingDirectoryReader.class);
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, engineFactory, Collections.emptyMap());
         module.setSearcherWrapper((s) -> new Wrapper());
 
         IndexService indexService = newIndexService(module);
@@ -164,18 +168,12 @@ public class IndexModuleTests extends ESTestCase {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "foo_store")
             .build();
-        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
-        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory());
-        module.addIndexStore("foo_store", FooStore::new);
-        try {
-            module.addIndexStore("foo_store", FooStore::new);
-            fail("already registered");
-        } catch (IllegalArgumentException ex) {
-            // fine
-        }
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
+        final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories = Collections.singletonMap("foo_store", FooStore::new);
+        final IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), indexStoreFactories);
 
-        IndexService indexService = newIndexService(module);
-        assertTrue(indexService.getIndexStore() instanceof FooStore);
+        final IndexService indexService = newIndexService(module);
+        assertThat(indexService.getIndexStore(), instanceOf(FooStore.class));
 
         indexService.close("simon says", false);
     }
@@ -189,7 +187,7 @@ public class IndexModuleTests extends ESTestCase {
             }
         };
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
-        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory());
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         module.addIndexEventListener(eventListener);
         IndexService indexService = newIndexService(module);
         IndexSettings x = indexService.getIndexSettings();
@@ -204,7 +202,7 @@ public class IndexModuleTests extends ESTestCase {
     public void testListener() throws IOException {
         Setting<Boolean> booleanSetting = Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope);
         final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings, booleanSetting);
-        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory());
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         Setting<Boolean> booleanSetting2 = Setting.boolSetting("index.foo.bar.baz", false, Property.Dynamic, Property.IndexScope);
         AtomicBoolean atomicBoolean = new AtomicBoolean(false);
         module.addSettingsUpdateConsumer(booleanSetting, atomicBoolean::set);
@@ -223,8 +221,8 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     public void testAddIndexOperationListener() throws IOException {
-        IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), emptyAnalysisRegistry, new InternalEngineFactory());
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         AtomicBoolean executed = new AtomicBoolean(false);
         IndexingOperationListener listener = new IndexingOperationListener() {
             @Override
@@ -254,8 +252,8 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     public void testAddSearchOperationListener() throws IOException {
-        IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), emptyAnalysisRegistry, new InternalEngineFactory());
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         AtomicBoolean executed = new AtomicBoolean(false);
         SearchOperationListener listener = new SearchOperationListener() {
 
@@ -288,8 +286,9 @@ public class IndexModuleTests extends ESTestCase {
                 .put("index.similarity.my_similarity.key", "there is a key")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
         IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings("foo", settings), emptyAnalysisRegistry, new InternalEngineFactory());
+                new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         module.addSimilarity("test_similarity",
                 (providerSettings, indexCreatedVersion, scriptService) -> new TestSimilarity(providerSettings.get("key")));
 
@@ -303,8 +302,8 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     public void testFrozen() {
-        IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), emptyAnalysisRegistry, new InternalEngineFactory());
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         module.freeze();
         String msg = "Can't modify IndexModule once the index service has been created";
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addSearchOperationListener(null)).getMessage());
@@ -313,7 +312,6 @@ public class IndexModuleTests extends ESTestCase {
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addSimilarity(null, null)).getMessage());
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.setSearcherWrapper(null)).getMessage());
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.forceQueryCacheProvider(null)).getMessage());
-        assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addIndexStore("foo", null)).getMessage());
     }
 
     public void testSetupUnknownSimilarity() throws IOException {
@@ -322,8 +320,9 @@ public class IndexModuleTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
         IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings("foo", settings), emptyAnalysisRegistry, new InternalEngineFactory());
+                new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         Exception ex = expectThrows(IllegalArgumentException.class, () -> newIndexService(module));
         assertEquals("Unknown Similarity type [test_similarity] for [my_similarity]", ex.getMessage());
     }
@@ -334,8 +333,8 @@ public class IndexModuleTests extends ESTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
-        IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings("foo", settings), emptyAnalysisRegistry, new InternalEngineFactory());
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         Exception ex = expectThrows(IllegalArgumentException.class, () -> newIndexService(module));
         assertEquals("Similarity [my_similarity] must have an associated type", ex.getMessage());
     }
@@ -344,8 +343,8 @@ public class IndexModuleTests extends ESTestCase {
         Settings settings = Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings("foo", settings), emptyAnalysisRegistry, new InternalEngineFactory());
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
         expectThrows(AlreadySetException.class, () -> module.forceQueryCacheProvider((a, b) -> new CustomQueryCache()));
         IndexService indexService = newIndexService(module);
@@ -357,8 +356,8 @@ public class IndexModuleTests extends ESTestCase {
         Settings settings = Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings("foo", settings), emptyAnalysisRegistry, new InternalEngineFactory());
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         IndexService indexService = newIndexService(module);
         assertTrue(indexService.cache().query() instanceof IndexQueryCache);
         indexService.close("simon says", false);
@@ -369,8 +368,8 @@ public class IndexModuleTests extends ESTestCase {
             .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false)
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexModule module =
-                new IndexModule(IndexSettingsModule.newIndexSettings("foo", settings), emptyAnalysisRegistry, new InternalEngineFactory());
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
+        IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
         module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
         IndexService indexService = newIndexService(module);
         assertTrue(indexService.cache().query() instanceof DisabledQueryCache);
diff --git a/server/src/test/java/org/elasticsearch/plugins/IndexStorePluginTests.java b/server/src/test/java/org/elasticsearch/plugins/IndexStorePluginTests.java
new file mode 100644
index 00000000000..c53d798f7b4
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/plugins/IndexStorePluginTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.bootstrap.JavaVersion;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.store.IndexStore;
+import org.elasticsearch.node.MockNode;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.Function;
+
+import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
+import static org.hamcrest.Matchers.hasToString;
+
+public class IndexStorePluginTests extends ESTestCase {
+
+    public static class BarStorePlugin extends Plugin implements IndexStorePlugin {
+
+        @Override
+        public Map<String, Function<IndexSettings, IndexStore>> getIndexStoreFactories() {
+            return Collections.singletonMap("store", IndexStore::new);
+        }
+
+    }
+
+    public static class FooStorePlugin extends Plugin implements IndexStorePlugin {
+
+        @Override
+        public Map<String, Function<IndexSettings, IndexStore>> getIndexStoreFactories() {
+            return Collections.singletonMap("store", IndexStore::new);
+        }
+
+    }
+
+    public void testDuplicateIndexStoreProviders() {
+        final Settings settings = Settings.builder().put("path.home", createTempDir()).build();
+        final IllegalStateException e = expectThrows(
+                IllegalStateException.class, () -> new MockNode(settings, Arrays.asList(BarStorePlugin.class, FooStorePlugin.class)));
+        if (JavaVersion.current().compareTo(JavaVersion.parse("9")) >= 0) {
+            assertThat(e, hasToString(matches(
+                    "java.lang.IllegalStateException: Duplicate key store \\(attempted merging values " +
+                            "org.elasticsearch.plugins.IndexStorePluginTests\\$BarStorePlugin.* " +
+                            "and org.elasticsearch.plugins.IndexStorePluginTests\\$FooStorePlugin.*\\)")));
+        } else {
+            assertThat(e, hasToString(matches(
+                    "java.lang.IllegalStateException: Duplicate key org.elasticsearch.plugins.IndexStorePluginTests\\$BarStorePlugin.*")));
+        }
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
index 921b819b9b7..8a22383dcae 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
@@ -34,6 +34,7 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.IndexStore;
+import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.plugins.Plugin;
 
 import java.util.Arrays;
@@ -42,13 +43,14 @@ import java.util.EnumSet;
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 public class MockFSIndexStore extends IndexStore {
 
     public static final Setting<Boolean> INDEX_CHECK_INDEX_ON_CLOSE_SETTING =
         Setting.boolSetting("index.store.mock.check_index_on_close", true, Property.IndexScope, Property.NodeScope);
 
-    public static class TestPlugin extends Plugin {
+    public static class TestPlugin extends Plugin implements IndexStorePlugin {
         @Override
         public Settings additionalSettings() {
             return Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mock").build();
@@ -64,6 +66,11 @@ public class MockFSIndexStore extends IndexStore {
             MockFSDirectoryService.RANDOM_IO_EXCEPTION_RATE_ON_OPEN_SETTING);
         }
 
+        @Override
+        public Map<String, Function<IndexSettings, IndexStore>> getIndexStoreFactories() {
+            return Collections.singletonMap("mock", MockFSIndexStore::new);
+        }
+
         @Override
         public void onIndexModule(IndexModule indexModule) {
             Settings indexSettings = indexModule.getSettings();
@@ -71,7 +78,6 @@ public class MockFSIndexStore extends IndexStore {
                 if (INDEX_CHECK_INDEX_ON_CLOSE_SETTING.get(indexSettings)) {
                     indexModule.addIndexEventListener(new Listener());
                 }
-                indexModule.addIndexStore("mock", MockFSIndexStore::new);
             }
         }
     }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
index 474f69c70ed..e345e890db1 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
@@ -17,6 +17,7 @@ import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.notification.NotificationService;
 
+import java.util.Collections;
 import java.util.List;
 
 import static java.util.Collections.emptyMap;
@@ -74,7 +75,7 @@ public class WatcherPluginTests extends ESTestCase {
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(Watch.INDEX, settings);
         AnalysisRegistry registry = new AnalysisRegistry(TestEnvironment.newEnvironment(settings), emptyMap(), emptyMap(), emptyMap(),
                 emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap());
-        IndexModule indexModule = new IndexModule(indexSettings, registry, new InternalEngineFactory());
+        IndexModule indexModule = new IndexModule(indexSettings, registry, new InternalEngineFactory(), Collections.emptyMap());
         // this will trip an assertion if the watcher indexing operation listener is null (which it is) but we try to add it
         watcher.onIndexModule(indexModule);
 

From 35ae87125d5fb5ff3395e43ada49b8a81cdf9a3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <cbuescher@posteo.de>
Date: Thu, 26 Jul 2018 17:12:51 +0200
Subject: [PATCH 191/260] Remove some dead code (#31993)

Removing some dead code or supressing warnings where apropriate. Most of the
time the variable tested for null is dereferenced earlier or never used before.
---
 .../support/ArrayValuesSourceParser.java      |  4 --
 .../support/nodes/TransportNodesAction.java   | 57 +++++++++----------
 .../common/geo/parsers/GeoJsonParser.java     |  6 +-
 .../blobstore/BlobStoreRepository.java        |  7 +--
 .../autodetect/writer/FieldConfigWriter.java  | 16 ++++--
 5 files changed, 39 insertions(+), 51 deletions(-)

diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java
index 1100884cf8a..acfbc04f872 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceParser.java
@@ -78,7 +78,6 @@ public abstract class ArrayValuesSourceParser<VS extends ValuesSource> implement
         throws IOException {
 
         List<String> fields = null;
-        ValueType valueType = null;
         String format = null;
         Map<String, Object> missingMap = null;
         Map<ParseField, Object> otherOptions = new HashMap<>();
@@ -145,9 +144,6 @@ public abstract class ArrayValuesSourceParser<VS extends ValuesSource> implement
         if (fields != null) {
             factory.fields(fields);
         }
-        if (valueType != null) {
-            factory.valueType(valueType);
-        }
         if (format != null) {
             factory.format(format);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index b232d849223..2be4e5bf053 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.action.support.nodes;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
-import org.elasticsearch.action.NoSuchNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.ClusterState;
@@ -179,37 +178,33 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
                 final DiscoveryNode node = nodes[i];
                 final String nodeId = node.getId();
                 try {
-                    if (node == null) {
-                        onFailure(idx, nodeId, new NoSuchNodeException(nodeId));
-                    } else {
-                        TransportRequest nodeRequest = newNodeRequest(nodeId, request);
-                        if (task != null) {
-                            nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                        }
-
-                        transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
-                                                     new TransportResponseHandler<NodeResponse>() {
-                            @Override
-                            public NodeResponse newInstance() {
-                                return newNodeResponse();
-                            }
-
-                            @Override
-                            public void handleResponse(NodeResponse response) {
-                                onOperation(idx, response);
-                            }
-
-                            @Override
-                            public void handleException(TransportException exp) {
-                                onFailure(idx, node.getId(), exp);
-                            }
-
-                            @Override
-                            public String executor() {
-                                return ThreadPool.Names.SAME;
-                            }
-                        });
+                    TransportRequest nodeRequest = newNodeRequest(nodeId, request);
+                    if (task != null) {
+                        nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
                     }
+
+                    transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
+                            new TransportResponseHandler<NodeResponse>() {
+                                @Override
+                                public NodeResponse newInstance() {
+                                    return newNodeResponse();
+                                }
+
+                                @Override
+                                public void handleResponse(NodeResponse response) {
+                                    onOperation(idx, response);
+                                }
+
+                                @Override
+                                public void handleException(TransportException exp) {
+                                    onFailure(idx, node.getId(), exp);
+                                }
+
+                                @Override
+                                public String executor() {
+                                    return ThreadPool.Names.SAME;
+                                }
+                            });
                 } catch (Exception e) {
                     onFailure(idx, nodeId, e);
                 }
diff --git a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java
index af0e0248471..45ce2b610ca 100644
--- a/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java
+++ b/server/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.common.geo.parsers;
 
-import org.locationtech.jts.geom.Coordinate;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -29,6 +28,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
+import org.locationtech.jts.geom.Coordinate;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -130,10 +130,6 @@ abstract class GeoJsonParser {
                 CircleBuilder.TYPE);
         }
 
-        if (shapeType == null) {
-            throw new ElasticsearchParseException("shape type [{}] not included", shapeType);
-        }
-
         if (shapeType.equals(GeoShapeType.GEOMETRYCOLLECTION)) {
             return geometryCollections;
         }
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 22743e38839..cc1d27425e1 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -1554,11 +1554,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                     filesToRecover.add(fileInfo);
                     recoveryState.getIndex().addFileDetail(fileInfo.name(), fileInfo.length(), false);
                     if (logger.isTraceEnabled()) {
-                        if (md == null) {
-                            logger.trace("[{}] [{}] recovering [{}] from [{}], does not exists in local store", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
-                        } else {
-                            logger.trace("[{}] [{}] recovering [{}] from [{}], exists in local store but is different", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
-                        }
+                        logger.trace("[{}] [{}] recovering [{}] from [{}], exists in local store but is different", shardId, snapshotId,
+                                fileInfo.physicalName(), fileInfo.name());
                     }
                 }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java
index a8c33dd98b6..f26027b3a07 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java
@@ -10,7 +10,6 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DefaultDetectorDescription;
@@ -18,6 +17,7 @@ import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;
 import org.elasticsearch.xpack.core.ml.job.config.Detector;
 import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
 import org.elasticsearch.xpack.core.ml.utils.MlStrings;
+import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.io.IOException;
 import java.io.OutputStreamWriter;
@@ -66,11 +66,7 @@ public class FieldConfigWriter {
         writeFilters(contents);
         writeDetectors(contents);
         writeScheduledEvents(contents);
-
-        if (MachineLearning.CATEGORIZATION_TOKENIZATION_IN_JAVA == false) {
-            writeAsEnumeratedSettings(CATEGORIZATION_FILTER_PREFIX, config.getCategorizationFilters(),
-                    contents, true);
-        }
+        writeCategorizationFilters(contents);
 
         // As values are written as entire settings rather than part of a
         // clause no quoting is needed
@@ -80,6 +76,14 @@ public class FieldConfigWriter {
         writer.write(contents.toString());
     }
 
+    @SuppressWarnings("unused") // CATEGORIZATION_TOKENIZATION_IN_JAVA is used for performance testing
+    private void writeCategorizationFilters(StringBuilder contents) {
+        if (MachineLearning.CATEGORIZATION_TOKENIZATION_IN_JAVA == false) {
+            writeAsEnumeratedSettings(CATEGORIZATION_FILTER_PREFIX, config.getCategorizationFilters(),
+                    contents, true);
+        }
+    }
+
     private void writeDetectors(StringBuilder contents) throws IOException {
         int counter = 0;
         for (Detector detector : config.getDetectors()) {

From 63a043676461d904b7447a1f494a1243ed154994 Mon Sep 17 00:00:00 2001
From: mzbeck <39967352+mzbeck@users.noreply.github.com>
Date: Thu, 26 Jul 2018 08:15:10 -0700
Subject: [PATCH 192/260] Update update-settings.asciidoc (#31378)

---
 .../cluster/update-settings.asciidoc          | 64 ++++++++++---------
 1 file changed, 34 insertions(+), 30 deletions(-)

diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc
index 57be634e835..27d5cc316a6 100644
--- a/docs/reference/cluster/update-settings.asciidoc
+++ b/docs/reference/cluster/update-settings.asciidoc
@@ -1,9 +1,18 @@
 [[cluster-update-settings]]
 == Cluster Update Settings
 
-Allows to update cluster wide specific settings. Settings updated can
-either be persistent (applied across restarts) or transient (will not
-survive a full cluster restart). Here is an example:
+Use this API to review and change cluster-wide settings. 
+
+To review cluster settings:
+
+[source,js]
+--------------------------------------------------
+GET /_cluster/settings
+--------------------------------------------------
+// CONSOLE
+
+Updates to settings can be persistent, meaning they apply across restarts, or transient, where they don't 
+survive a full cluster restart. Here is an example of a persistent update:
 
 [source,js]
 --------------------------------------------------
@@ -16,7 +25,7 @@ PUT /_cluster/settings
 --------------------------------------------------
 // CONSOLE
 
-Or:
+This update is transient:
 
 [source,js]
 --------------------------------------------------
@@ -29,8 +38,7 @@ PUT /_cluster/settings?flat_settings=true
 --------------------------------------------------
 // CONSOLE
 
-The cluster responds with the settings updated. So the response for the
-last example will be:
+The response to an update returns the changed setting, as in this response to the transient example:
 
 [source,js]
 --------------------------------------------------
@@ -44,11 +52,14 @@ last example will be:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"acknowledged": true,/]
 
-Resetting persistent or transient settings can be done by assigning a
-`null` value. If a transient setting is reset, the persistent setting
-is applied if available. Otherwise Elasticsearch will fallback to the setting
-defined at the configuration file or, if not existent, to the default
-value. Here is an example:
+You can reset persistent or transient settings by assigning a
+`null` value. If a transient setting is reset, the first one of these values that is defined is applied:
+
+* the persistent setting
+* the setting in the configuration file
+* the default value. 
+
+This example resets a setting:
 
 [source,js]
 --------------------------------------------------
@@ -61,8 +72,7 @@ PUT /_cluster/settings
 --------------------------------------------------
 // CONSOLE
 
-Reset settings will not be included in the cluster response. So
-the response for the last example will be:
+The response does not include settings that have been reset:
 
 [source,js]
 --------------------------------------------------
@@ -74,8 +84,8 @@ the response for the last example will be:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"acknowledged": true,/]
 
-Settings can also be reset using simple wildcards. For instance to reset
-all dynamic `indices.recovery` setting a prefix can be used:
+You can also reset settings using wildcards. For example, to reset
+all dynamic `indices.recovery` settings:
 
 [source,js]
 --------------------------------------------------
@@ -88,25 +98,19 @@ PUT /_cluster/settings
 --------------------------------------------------
 // CONSOLE
 
-Cluster wide settings can be returned using:
-
-[source,js]
---------------------------------------------------
-GET /_cluster/settings
---------------------------------------------------
-// CONSOLE
 
 [float]
-=== Precedence of settings
+=== Order of Precedence
 
-Transient cluster settings take precedence over persistent cluster settings,
-which take precedence over settings configured in the `elasticsearch.yml`
-config file.
+The order of precedence for cluster settings is:
 
-For this reason it is preferrable to  use the `elasticsearch.yml` file only
-for local configurations, and set all cluster-wider settings with the
+1. transient cluster settings 
+2. persistent cluster settings
+3. settings in the `elasticsearch.yml` configuration file.
+
+It's best to use the `elasticsearch.yml` file only
+for local configurations, and set all cluster-wide settings with the
 `settings` API.
 
-A list of dynamically updatable settings can be found in the
-<<modules,Modules>> documentation.
+You can find the list of settings that you can dynamically update in <<modules,Modules>>.
 

From 0afa265ac90277b0b21dc125da29181550b7e99e Mon Sep 17 00:00:00 2001
From: David Roberts <dave.roberts@elastic.co>
Date: Thu, 26 Jul 2018 16:55:40 +0100
Subject: [PATCH 193/260] [ML] Consistent pattern for strict/lenient parser
 names (#32399)

Previously we had two patterns for naming of strict
and lenient parsers.

Some classes had CONFIG_PARSER and METADATA_PARSER,
and used an enum to pass the parser type to nested
parsers.

Other classes had STRICT_PARSER and LENIENT_PARSER
and used ternary operators to pass the parser type
to nested parsers.

This change makes all ML classes use the second of
the patterns described above.
---
 .../xpack/core/XPackClientPlugin.java         |   2 +-
 .../xpack/core/ml/MlMetadata.java             |   8 +-
 .../xpack/core/ml/MlParserType.java           |  19 ---
 .../core/ml/action/PutDatafeedAction.java     |   2 +-
 .../xpack/core/ml/action/PutJobAction.java    |   2 +-
 .../ml/action/ValidateDetectorAction.java     |   2 +-
 .../ml/action/ValidateJobConfigAction.java    |   2 +-
 .../core/ml/datafeed/ChunkingConfig.java      |  48 +++----
 .../core/ml/datafeed/DatafeedConfig.java      |  76 ++++++------
 .../core/ml/datafeed/DatafeedUpdate.java      |   2 +-
 .../core/ml/job/config/AnalysisConfig.java    |  73 +++++------
 .../core/ml/job/config/AnalysisLimits.java    |  46 +++----
 .../config/CategorizationAnalyzerConfig.java  |  10 +-
 .../core/ml/job/config/DataDescription.java   |  33 ++---
 .../core/ml/job/config/DetectionRule.java     |  30 ++---
 .../xpack/core/ml/job/config/Detector.java    |  53 ++++----
 .../xpack/core/ml/job/config/FilterRef.java   |  39 +++---
 .../xpack/core/ml/job/config/Job.java         | 117 +++++++++---------
 .../xpack/core/ml/job/config/JobUpdate.java   |   6 +-
 .../core/ml/job/config/ModelPlotConfig.java   |  30 ++---
 .../core/ml/job/config/RuleCondition.java     |  52 ++++----
 .../xpack/core/ml/job/config/RuleScope.java   |   6 +-
 .../core/ml/datafeed/ChunkingConfigTests.java |   2 +-
 .../core/ml/datafeed/DatafeedConfigTests.java |   6 +-
 .../ml/job/config/AnalysisConfigTests.java    |   2 +-
 .../ml/job/config/AnalysisLimitsTests.java    |  18 +--
 .../ml/job/config/DataDescriptionTests.java   |   8 +-
 .../ml/job/config/DetectionRuleTests.java     |   2 +-
 .../core/ml/job/config/DetectorTests.java     |   2 +-
 .../core/ml/job/config/FilterRefTests.java    |   4 +-
 .../xpack/core/ml/job/config/JobTests.java    |   6 +-
 .../ml/job/config/ModelPlotConfigTests.java   |   2 +-
 .../ml/job/config/RuleConditionTests.java     |   2 +-
 .../xpack/ml/MlMetadataTests.java             |   2 +-
 .../CategorizationAnalyzerConfigTests.java    |   3 +-
 .../xpack/ml/job/config/JobBuilderTests.java  |   2 +-
 36 files changed, 311 insertions(+), 408 deletions(-)
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlParserType.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index 6e57c243630..0389ceffbc3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -373,7 +373,7 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
         return Arrays.asList(
                 // ML - Custom metadata
                 new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField("ml"),
-                        parser -> MlMetadata.METADATA_PARSER.parse(parser, null).build()),
+                        parser -> MlMetadata.LENIENT_PARSER.parse(parser, null).build()),
                 // ML - Persistent action requests
                 new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(StartDatafeedAction.TASK_NAME),
                         StartDatafeedAction.DatafeedParams::fromXContent),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
index f3505835994..e0b71abe966 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
@@ -61,12 +61,12 @@ public class MlMetadata implements XPackPlugin.XPackMetaDataCustom {
 
     public static final MlMetadata EMPTY_METADATA = new MlMetadata(Collections.emptySortedMap(), Collections.emptySortedMap());
     // This parser follows the pattern that metadata is parsed leniently (to allow for enhancements)
-    public static final ObjectParser<Builder, Void> METADATA_PARSER = new ObjectParser<>("ml_metadata", true, Builder::new);
+    public static final ObjectParser<Builder, Void> LENIENT_PARSER = new ObjectParser<>("ml_metadata", true, Builder::new);
 
     static {
-        METADATA_PARSER.declareObjectArray(Builder::putJobs, (p, c) -> Job.METADATA_PARSER.apply(p, c).build(), JOBS_FIELD);
-        METADATA_PARSER.declareObjectArray(Builder::putDatafeeds,
-                (p, c) -> DatafeedConfig.METADATA_PARSER.apply(p, c).build(), DATAFEEDS_FIELD);
+        LENIENT_PARSER.declareObjectArray(Builder::putJobs, (p, c) -> Job.LENIENT_PARSER.apply(p, c).build(), JOBS_FIELD);
+        LENIENT_PARSER.declareObjectArray(Builder::putDatafeeds,
+                (p, c) -> DatafeedConfig.LENIENT_PARSER.apply(p, c).build(), DATAFEEDS_FIELD);
     }
 
     private final SortedMap<String, Job> jobs;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlParserType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlParserType.java
deleted file mode 100644
index 64f52ab2d2c..00000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlParserType.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.ml;
-
-/**
- * In order to allow enhancements that require additions to the ML custom cluster state to be made in minor versions,
- * when we parse our metadata from persisted cluster state we ignore unknown fields.  However, we don't want to be
- * lenient when parsing config as this would mean user mistakes could go undetected.  Therefore, for all JSON objects
- * that are used in both custom cluster state and config we have two parsers, one tolerant of unknown fields (for
- * parsing cluster state) and one strict (for parsing config).  This class enumerates the two options.
- */
-public enum MlParserType {
-
-    METADATA, CONFIG;
-
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java
index a0c757a0be6..4d3f720026e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java
@@ -39,7 +39,7 @@ public class PutDatafeedAction extends Action<PutDatafeedAction.Response> {
     public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
 
         public static Request parseRequest(String datafeedId, XContentParser parser) {
-            DatafeedConfig.Builder datafeed = DatafeedConfig.CONFIG_PARSER.apply(parser, null);
+            DatafeedConfig.Builder datafeed = DatafeedConfig.STRICT_PARSER.apply(parser, null);
             datafeed.setId(datafeedId);
             return new Request(datafeed.build());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutJobAction.java
index 7e85198d214..f241e4bd375 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutJobAction.java
@@ -42,7 +42,7 @@ public class PutJobAction extends Action<PutJobAction.Response> {
     public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
 
         public static Request parseRequest(String jobId, XContentParser parser) {
-            Job.Builder jobBuilder = Job.CONFIG_PARSER.apply(parser, null);
+            Job.Builder jobBuilder = Job.STRICT_PARSER.apply(parser, null);
             if (jobBuilder.getId() == null) {
                 jobBuilder.setId(jobId);
             } else if (!Strings.isNullOrEmpty(jobId) && !jobId.equals(jobBuilder.getId())) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorAction.java
index 0e807664d86..15dd8721f06 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorAction.java
@@ -48,7 +48,7 @@ public class ValidateDetectorAction extends Action<ValidateDetectorAction.Respon
         private Detector detector;
 
         public static Request parseRequest(XContentParser parser) {
-            Detector detector = Detector.CONFIG_PARSER.apply(parser, null).build();
+            Detector detector = Detector.STRICT_PARSER.apply(parser, null).build();
             return new Request(detector);
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigAction.java
index 00a8813c1f1..4e77882c1fc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigAction.java
@@ -49,7 +49,7 @@ public class ValidateJobConfigAction extends Action<ValidateJobConfigAction.Resp
         private Job job;
 
         public static Request parseRequest(XContentParser parser) {
-            Job.Builder job = Job.CONFIG_PARSER.apply(parser, null);
+            Job.Builder job = Job.STRICT_PARSER.apply(parser, null);
             // When jobs are PUT their ID must be supplied in the URL - assume this will
             // be valid unless an invalid job ID is specified in the JSON to be validated
             job.setId(job.getId() != null ? job.getId() : "ok");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfig.java
index 43adcf4d63a..51aa8c147e7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfig.java
@@ -16,13 +16,10 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
-import java.util.EnumMap;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -34,32 +31,27 @@ public class ChunkingConfig implements ToXContentObject, Writeable {
     public static final ParseField TIME_SPAN_FIELD = new ParseField("time_span");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ConstructingObjectParser<ChunkingConfig, Void> METADATA_PARSER = new ConstructingObjectParser<>(
-            "chunking_config", true, a -> new ChunkingConfig((Mode) a[0], (TimeValue) a[1]));
-    public static final ConstructingObjectParser<ChunkingConfig, Void> CONFIG_PARSER = new ConstructingObjectParser<>(
-            "chunking_config", false, a -> new ChunkingConfig((Mode) a[0], (TimeValue) a[1]));
-    public static final Map<MlParserType, ConstructingObjectParser<ChunkingConfig, Void>> PARSERS =
-            new EnumMap<>(MlParserType.class);
+    public static final ConstructingObjectParser<ChunkingConfig, Void> LENIENT_PARSER = createParser(true);
+    public static final ConstructingObjectParser<ChunkingConfig, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ConstructingObjectParser<ChunkingConfig, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareField(ConstructingObjectParser.constructorArg(), p -> {
-                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                    return Mode.fromString(p.text());
-                }
-                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-            }, MODE_FIELD, ValueType.STRING);
-            parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
-                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                    return TimeValue.parseTimeValue(p.text(), TIME_SPAN_FIELD.getPreferredName());
-                }
-                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-            }, TIME_SPAN_FIELD, ValueType.STRING);
-        }
+    private static ConstructingObjectParser<ChunkingConfig, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<ChunkingConfig, Void> parser = new ConstructingObjectParser<>(
+            "chunking_config", ignoreUnknownFields, a -> new ChunkingConfig((Mode) a[0], (TimeValue) a[1]));
+
+        parser.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return Mode.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, MODE_FIELD, ValueType.STRING);
+        parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return TimeValue.parseTimeValue(p.text(), TIME_SPAN_FIELD.getPreferredName());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, TIME_SPAN_FIELD, ValueType.STRING);
+
+        return parser;
     }
 
     private final Mode mode;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
index e9ac704171b..1034b00af0a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
@@ -25,7 +25,6 @@ import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
@@ -38,7 +37,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -87,44 +85,46 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
     public static final ParseField HEADERS = new ParseField("headers");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ObjectParser<Builder, Void> METADATA_PARSER = new ObjectParser<>("datafeed_config", true, Builder::new);
-    public static final ObjectParser<Builder, Void> CONFIG_PARSER = new ObjectParser<>("datafeed_config", false, Builder::new);
-    public static final Map<MlParserType, ObjectParser<Builder, Void>> PARSERS = new EnumMap<>(MlParserType.class);
+    public static final ObjectParser<Builder, Void> LENIENT_PARSER = createParser(true);
+    public static final ObjectParser<Builder, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ObjectParser<Builder, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareString(Builder::setId, ID);
-            parser.declareString(Builder::setJobId, Job.ID);
-            parser.declareStringArray(Builder::setIndices, INDEXES);
-            parser.declareStringArray(Builder::setIndices, INDICES);
-            parser.declareStringArray(Builder::setTypes, TYPES);
-            parser.declareString((builder, val) ->
-                    builder.setQueryDelay(TimeValue.parseTimeValue(val, QUERY_DELAY.getPreferredName())), QUERY_DELAY);
-            parser.declareString((builder, val) ->
-                    builder.setFrequency(TimeValue.parseTimeValue(val, FREQUENCY.getPreferredName())), FREQUENCY);
-            parser.declareObject(Builder::setQuery, (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), QUERY);
-            parser.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p), AGGREGATIONS);
-            parser.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p), AGGS);
-            parser.declareObject(Builder::setScriptFields, (p, c) -> {
-                List<SearchSourceBuilder.ScriptField> parsedScriptFields = new ArrayList<>();
-                while (p.nextToken() != XContentParser.Token.END_OBJECT) {
-                    parsedScriptFields.add(new SearchSourceBuilder.ScriptField(p));
-                }
-                parsedScriptFields.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName));
-                return parsedScriptFields;
-            }, SCRIPT_FIELDS);
-            parser.declareInt(Builder::setScrollSize, SCROLL_SIZE);
-            // TODO this is to read former _source field. Remove in v7.0.0
-            parser.declareBoolean((builder, value) -> {}, SOURCE);
-            parser.declareObject(Builder::setChunkingConfig, ChunkingConfig.PARSERS.get(parserType), CHUNKING_CONFIG);
+    private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFields) {
+        ObjectParser<Builder, Void> parser = new ObjectParser<>("datafeed_config", ignoreUnknownFields, Builder::new);
+
+        parser.declareString(Builder::setId, ID);
+        parser.declareString(Builder::setJobId, Job.ID);
+        parser.declareStringArray(Builder::setIndices, INDEXES);
+        parser.declareStringArray(Builder::setIndices, INDICES);
+        parser.declareStringArray(Builder::setTypes, TYPES);
+        parser.declareString((builder, val) ->
+            builder.setQueryDelay(TimeValue.parseTimeValue(val, QUERY_DELAY.getPreferredName())), QUERY_DELAY);
+        parser.declareString((builder, val) ->
+            builder.setFrequency(TimeValue.parseTimeValue(val, FREQUENCY.getPreferredName())), FREQUENCY);
+        parser.declareObject(Builder::setQuery, (p, c) -> AbstractQueryBuilder.parseInnerQueryBuilder(p), QUERY);
+        parser.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p), AGGREGATIONS);
+        parser.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(p), AGGS);
+        parser.declareObject(Builder::setScriptFields, (p, c) -> {
+            List<SearchSourceBuilder.ScriptField> parsedScriptFields = new ArrayList<>();
+            while (p.nextToken() != XContentParser.Token.END_OBJECT) {
+                parsedScriptFields.add(new SearchSourceBuilder.ScriptField(p));
+            }
+            parsedScriptFields.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName));
+            return parsedScriptFields;
+        }, SCRIPT_FIELDS);
+        parser.declareInt(Builder::setScrollSize, SCROLL_SIZE);
+        // TODO this is to read former _source field. Remove in v7.0.0
+        parser.declareBoolean((builder, value) -> {
+        }, SOURCE);
+        parser.declareObject(Builder::setChunkingConfig, ignoreUnknownFields ? ChunkingConfig.LENIENT_PARSER : ChunkingConfig.STRICT_PARSER,
+            CHUNKING_CONFIG);
+
+        if (ignoreUnknownFields) {
+            // Headers are not parsed by the strict (config) parser, so headers supplied in the _body_ of a REST request will be rejected.
+            // (For config, headers are explicitly transferred from the auth headers by code in the put/update datafeed actions.)
+            parser.declareObject(Builder::setHeaders, (p, c) -> p.mapStrings(), HEADERS);
         }
-        // Headers are only parsed by the metadata parser, so headers supplied in the _body_ of a REST request will be rejected.
-        // (For config headers are explicitly transferred from the auth headers by code in the put/update datafeed actions.)
-        METADATA_PARSER.declareObject(Builder::setHeaders, (p, c) -> p.mapStrings(), HEADERS);
+
+        return parser;
     }
 
     private final String id;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java
index 5d8fd3ffc71..f3748cefc51 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java
@@ -68,7 +68,7 @@ public class DatafeedUpdate implements Writeable, ToXContentObject {
             return parsedScriptFields;
         }, DatafeedConfig.SCRIPT_FIELDS);
         PARSER.declareInt(Builder::setScrollSize, DatafeedConfig.SCROLL_SIZE);
-        PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.CONFIG_PARSER, DatafeedConfig.CHUNKING_CONFIG);
+        PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.STRICT_PARSER, DatafeedConfig.CHUNKING_CONFIG);
     }
 
     private final String id;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
index 0c702e5afb0..371cdd8c038 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
@@ -16,7 +16,6 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
@@ -24,10 +23,8 @@ import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.EnumMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.SortedSet;
@@ -76,46 +73,38 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
     public static final long DEFAULT_RESULT_FINALIZATION_WINDOW = 2L;
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    @SuppressWarnings("unchecked")
-    public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> METADATA_PARSER =
-            new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(), true,
-                    a -> new AnalysisConfig.Builder((List<Detector>) a[0]));
-    @SuppressWarnings("unchecked")
-    public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> CONFIG_PARSER =
-            new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(), false,
-                    a -> new AnalysisConfig.Builder((List<Detector>) a[0]));
-    public static final Map<MlParserType, ConstructingObjectParser<Builder, Void>> PARSERS =
-            new EnumMap<>(MlParserType.class);
+    public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> LENIENT_PARSER = createParser(true);
+    public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ConstructingObjectParser<AnalysisConfig.Builder, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareObjectArray(ConstructingObjectParser.constructorArg(),
-                    (p, c) -> Detector.PARSERS.get(parserType).apply(p, c).build(), DETECTORS);
-            parser.declareString((builder, val) ->
-                    builder.setBucketSpan(TimeValue.parseTimeValue(val, BUCKET_SPAN.getPreferredName())), BUCKET_SPAN);
-            parser.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME);
-            parser.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS);
-            // This one is nasty - the syntax for analyzers takes either names or objects at many levels, hence it's not
-            // possible to simply declare whether the field is a string or object and a completely custom parser is required
-            parser.declareField(Builder::setCategorizationAnalyzerConfig,
-                    (p, c) -> CategorizationAnalyzerConfig.buildFromXContentFragment(p, parserType),
-                    CATEGORIZATION_ANALYZER, ObjectParser.ValueType.OBJECT_OR_STRING);
-            parser.declareString((builder, val) ->
-                    builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
-            parser.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
-            parser.declareStringArray(Builder::setInfluencers, INFLUENCERS);
-            parser.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
-            parser.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
-            parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
-            parser.declareStringArray((builder, values) -> builder.setMultipleBucketSpans(
-                    values.stream().map(v -> TimeValue.parseTimeValue(v, MULTIPLE_BUCKET_SPANS.getPreferredName()))
-                            .collect(Collectors.toList())), MULTIPLE_BUCKET_SPANS);
-            parser.declareBoolean(Builder::setUsePerPartitionNormalization, USER_PER_PARTITION_NORMALIZATION);
-        }
+    @SuppressWarnings("unchecked")
+    private static ConstructingObjectParser<AnalysisConfig.Builder, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<AnalysisConfig.Builder, Void> parser = new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(),
+            ignoreUnknownFields, a -> new AnalysisConfig.Builder((List<Detector>) a[0]));
+
+        parser.declareObjectArray(ConstructingObjectParser.constructorArg(),
+            (p, c) -> (ignoreUnknownFields ? Detector.LENIENT_PARSER : Detector.STRICT_PARSER).apply(p, c).build(), DETECTORS);
+        parser.declareString((builder, val) ->
+            builder.setBucketSpan(TimeValue.parseTimeValue(val, BUCKET_SPAN.getPreferredName())), BUCKET_SPAN);
+        parser.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME);
+        parser.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS);
+        // This one is nasty - the syntax for analyzers takes either names or objects at many levels, hence it's not
+        // possible to simply declare whether the field is a string or object and a completely custom parser is required
+        parser.declareField(Builder::setCategorizationAnalyzerConfig,
+            (p, c) -> CategorizationAnalyzerConfig.buildFromXContentFragment(p, ignoreUnknownFields),
+            CATEGORIZATION_ANALYZER, ObjectParser.ValueType.OBJECT_OR_STRING);
+        parser.declareString((builder, val) ->
+            builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
+        parser.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
+        parser.declareStringArray(Builder::setInfluencers, INFLUENCERS);
+        parser.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
+        parser.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
+        parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
+        parser.declareStringArray((builder, values) -> builder.setMultipleBucketSpans(
+            values.stream().map(v -> TimeValue.parseTimeValue(v, MULTIPLE_BUCKET_SPANS.getPreferredName()))
+                .collect(Collectors.toList())), MULTIPLE_BUCKET_SPANS);
+        parser.declareBoolean(Builder::setUsePerPartitionNormalization, USER_PER_PARTITION_NORMALIZATION);
+
+        return parser;
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java
index 569d62a02cf..797df5892f8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimits.java
@@ -17,13 +17,10 @@ import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
-import java.util.EnumMap;
-import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -50,31 +47,26 @@ public class AnalysisLimits implements ToXContentObject, Writeable {
     public static final ParseField CATEGORIZATION_EXAMPLES_LIMIT = new ParseField("categorization_examples_limit");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ConstructingObjectParser<AnalysisLimits, Void> METADATA_PARSER = new ConstructingObjectParser<>(
-            "analysis_limits", true, a -> new AnalysisLimits(
-                    a[0] == null ? PRE_6_1_DEFAULT_MODEL_MEMORY_LIMIT_MB : (Long) a[0],
-                    a[1] == null ? DEFAULT_CATEGORIZATION_EXAMPLES_LIMIT : (Long) a[1]));
-    public static final ConstructingObjectParser<AnalysisLimits, Void> CONFIG_PARSER = new ConstructingObjectParser<>(
-            "analysis_limits", false, a -> new AnalysisLimits((Long) a[0], (Long) a[1]));
-    public static final Map<MlParserType, ConstructingObjectParser<AnalysisLimits, Void>> PARSERS =
-            new EnumMap<>(MlParserType.class);
+    public static final ConstructingObjectParser<AnalysisLimits, Void> LENIENT_PARSER = createParser(true);
+    public static final ConstructingObjectParser<AnalysisLimits, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ConstructingObjectParser<AnalysisLimits, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
-                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                    return ByteSizeValue.parseBytesSizeValue(p.text(), MODEL_MEMORY_LIMIT.getPreferredName()).getMb();
-                } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-                    return p.longValue();
-                }
-                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-            }, MODEL_MEMORY_LIMIT, ObjectParser.ValueType.VALUE);
-            parser.declareLong(ConstructingObjectParser.optionalConstructorArg(), CATEGORIZATION_EXAMPLES_LIMIT);
-        }
+    private static ConstructingObjectParser<AnalysisLimits, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<AnalysisLimits, Void> parser = new ConstructingObjectParser<>(
+            "analysis_limits", ignoreUnknownFields, a -> ignoreUnknownFields ? new AnalysisLimits(
+                a[0] == null ? PRE_6_1_DEFAULT_MODEL_MEMORY_LIMIT_MB : (Long) a[0],
+                a[1] == null ? DEFAULT_CATEGORIZATION_EXAMPLES_LIMIT : (Long) a[1]) : new AnalysisLimits((Long) a[0], (Long) a[1]));
+
+        parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ByteSizeValue.parseBytesSizeValue(p.text(), MODEL_MEMORY_LIMIT.getPreferredName()).getMb();
+            } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+                return p.longValue();
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, MODEL_MEMORY_LIMIT, ObjectParser.ValueType.VALUE);
+        parser.declareLong(ConstructingObjectParser.optionalConstructorArg(), CATEGORIZATION_EXAMPLES_LIMIT);
+
+        return parser;
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
index fd0fde76e68..36c25e0a7a7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
@@ -17,7 +17,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -61,7 +60,8 @@ public class CategorizationAnalyzerConfig implements ToXContentFragment, Writeab
     /**
      * This method is only used in the unit tests - in production code this config is always parsed as a fragment.
      */
-    public static CategorizationAnalyzerConfig buildFromXContentObject(XContentParser parser, MlParserType parserType) throws IOException {
+    public static CategorizationAnalyzerConfig buildFromXContentObject(XContentParser parser, boolean ignoreUnknownFields)
+        throws IOException {
 
         if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
             throw new IllegalArgumentException("Expected start object but got [" + parser.currentToken() + "]");
@@ -71,7 +71,7 @@ public class CategorizationAnalyzerConfig implements ToXContentFragment, Writeab
             throw new IllegalArgumentException("Expected [" + CATEGORIZATION_ANALYZER + "] field but got [" + parser.currentToken() + "]");
         }
         parser.nextToken();
-        CategorizationAnalyzerConfig categorizationAnalyzerConfig = buildFromXContentFragment(parser, parserType);
+        CategorizationAnalyzerConfig categorizationAnalyzerConfig = buildFromXContentFragment(parser, ignoreUnknownFields);
         parser.nextToken();
         return categorizationAnalyzerConfig;
     }
@@ -83,7 +83,7 @@ public class CategorizationAnalyzerConfig implements ToXContentFragment, Writeab
      *
      * The parser is strict when parsing config and lenient when parsing cluster state.
      */
-    static CategorizationAnalyzerConfig buildFromXContentFragment(XContentParser parser, MlParserType parserType) throws IOException {
+    static CategorizationAnalyzerConfig buildFromXContentFragment(XContentParser parser, boolean ignoreUnknownFields) throws IOException {
 
         CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder();
 
@@ -131,7 +131,7 @@ public class CategorizationAnalyzerConfig implements ToXContentFragment, Writeab
                         }
                     }
                 // Be lenient when parsing cluster state - assume unknown fields are from future versions
-                } else if (parserType == MlParserType.CONFIG) {
+                } else if (ignoreUnknownFields == false) {
                     throw new IllegalArgumentException("Parameter [" + currentFieldName + "] in [" + CATEGORIZATION_ANALYZER +
                             "] is unknown or of the wrong type [" + token + "]");
                 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java
index 6e9652bdfa2..87c084baeac 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java
@@ -14,16 +14,13 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.time.DateTimeFormatterTimestampConverter;
 
 import java.io.IOException;
 import java.time.ZoneOffset;
-import java.util.EnumMap;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -126,24 +123,20 @@ public class DataDescription implements ToXContentObject, Writeable {
     private final Character quoteCharacter;
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ObjectParser<Builder, Void> METADATA_PARSER =
-            new ObjectParser<>(DATA_DESCRIPTION_FIELD.getPreferredName(), true, Builder::new);
-    public static final ObjectParser<Builder, Void> CONFIG_PARSER =
-            new ObjectParser<>(DATA_DESCRIPTION_FIELD.getPreferredName(), false, Builder::new);
-    public static final Map<MlParserType, ObjectParser<Builder, Void>> PARSERS = new EnumMap<>(MlParserType.class);
+    public static final ObjectParser<Builder, Void> LENIENT_PARSER = createParser(true);
+    public static final ObjectParser<Builder, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ObjectParser<Builder, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareString(Builder::setFormat, FORMAT_FIELD);
-            parser.declareString(Builder::setTimeField, TIME_FIELD_NAME_FIELD);
-            parser.declareString(Builder::setTimeFormat, TIME_FORMAT_FIELD);
-            parser.declareField(Builder::setFieldDelimiter, DataDescription::extractChar, FIELD_DELIMITER_FIELD, ValueType.STRING);
-            parser.declareField(Builder::setQuoteCharacter, DataDescription::extractChar, QUOTE_CHARACTER_FIELD, ValueType.STRING);
-        }
+    private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFields) {
+        ObjectParser<Builder, Void> parser =
+            new ObjectParser<>(DATA_DESCRIPTION_FIELD.getPreferredName(), ignoreUnknownFields, Builder::new);
+
+        parser.declareString(Builder::setFormat, FORMAT_FIELD);
+        parser.declareString(Builder::setTimeField, TIME_FIELD_NAME_FIELD);
+        parser.declareString(Builder::setTimeFormat, TIME_FORMAT_FIELD);
+        parser.declareField(Builder::setFieldDelimiter, DataDescription::extractChar, FIELD_DELIMITER_FIELD, ValueType.STRING);
+        parser.declareField(Builder::setQuoteCharacter, DataDescription::extractChar, QUOTE_CHARACTER_FIELD, ValueType.STRING);
+
+        return parser;
     }
 
     public DataDescription(DataFormat dataFormat, String timeFieldName, String timeFormat, Character fieldDelimiter,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
index fbdb2f6662a..25cd0cffe7b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
@@ -13,17 +13,14 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumMap;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -37,23 +34,18 @@ public class DetectionRule implements ToXContentObject, Writeable {
     public static final ParseField CONDITIONS_FIELD = new ParseField("conditions");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ObjectParser<Builder, Void> METADATA_PARSER =
-            new ObjectParser<>(DETECTION_RULE_FIELD.getPreferredName(), true, Builder::new);
-    public static final ObjectParser<Builder, Void> CONFIG_PARSER =
-            new ObjectParser<>(DETECTION_RULE_FIELD.getPreferredName(), false, Builder::new);
-    public static final Map<MlParserType, ObjectParser<Builder, Void>> PARSERS = new EnumMap<>(MlParserType.class);
+    public static final ObjectParser<Builder, Void> LENIENT_PARSER = createParser(true);
+    public static final ObjectParser<Builder, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ObjectParser<Builder, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareStringArray(Builder::setActions, ACTIONS_FIELD);
-            parser.declareObject(Builder::setScope, RuleScope.parser(parserType), SCOPE_FIELD);
-            parser.declareObjectArray(Builder::setConditions, (p, c) ->
-                    RuleCondition.PARSERS.get(parserType).apply(p, c), CONDITIONS_FIELD);
-        }
+    private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFields) {
+        ObjectParser<Builder, Void> parser = new ObjectParser<>(DETECTION_RULE_FIELD.getPreferredName(), ignoreUnknownFields, Builder::new);
+
+        parser.declareStringArray(Builder::setActions, ACTIONS_FIELD);
+        parser.declareObject(Builder::setScope, RuleScope.parser(ignoreUnknownFields), SCOPE_FIELD);
+        parser.declareObjectArray(Builder::setConditions, ignoreUnknownFields ? RuleCondition.LENIENT_PARSER : RuleCondition.STRICT_PARSER,
+            CONDITIONS_FIELD);
+
+        return parser;
     }
 
     private final EnumSet<RuleAction> actions;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java
index dc4b55d73a5..93aa5495c40 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java
@@ -16,7 +16,6 @@ import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.writer.RecordWriter;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -26,12 +25,10 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumMap;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
@@ -89,33 +86,31 @@ public class Detector implements ToXContentObject, Writeable {
     public static final ParseField DETECTOR_INDEX = new ParseField("detector_index");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ObjectParser<Builder, Void> METADATA_PARSER = new ObjectParser<>("detector", true, Builder::new);
-    public static final ObjectParser<Builder, Void> CONFIG_PARSER = new ObjectParser<>("detector", false, Builder::new);
-    public static final Map<MlParserType, ObjectParser<Builder, Void>> PARSERS = new EnumMap<>(MlParserType.class);
+    public static final ObjectParser<Builder, Void> LENIENT_PARSER = createParser(true);
+    public static final ObjectParser<Builder, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ObjectParser<Builder, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareString(Builder::setDetectorDescription, DETECTOR_DESCRIPTION_FIELD);
-            parser.declareString(Builder::setFunction, FUNCTION_FIELD);
-            parser.declareString(Builder::setFieldName, FIELD_NAME_FIELD);
-            parser.declareString(Builder::setByFieldName, BY_FIELD_NAME_FIELD);
-            parser.declareString(Builder::setOverFieldName, OVER_FIELD_NAME_FIELD);
-            parser.declareString(Builder::setPartitionFieldName, PARTITION_FIELD_NAME_FIELD);
-            parser.declareBoolean(Builder::setUseNull, USE_NULL_FIELD);
-            parser.declareField(Builder::setExcludeFrequent, p -> {
-                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                    return ExcludeFrequent.forString(p.text());
-                }
-                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-            }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING);
-            parser.declareObjectArray(Builder::setRules, (p, c) ->
-                    DetectionRule.PARSERS.get(parserType).apply(p, c).build(), CUSTOM_RULES_FIELD);
-            parser.declareInt(Builder::setDetectorIndex, DETECTOR_INDEX);
-        }
+    private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFields) {
+        ObjectParser<Builder, Void> parser = new ObjectParser<>("detector", ignoreUnknownFields, Builder::new);
+
+        parser.declareString(Builder::setDetectorDescription, DETECTOR_DESCRIPTION_FIELD);
+        parser.declareString(Builder::setFunction, FUNCTION_FIELD);
+        parser.declareString(Builder::setFieldName, FIELD_NAME_FIELD);
+        parser.declareString(Builder::setByFieldName, BY_FIELD_NAME_FIELD);
+        parser.declareString(Builder::setOverFieldName, OVER_FIELD_NAME_FIELD);
+        parser.declareString(Builder::setPartitionFieldName, PARTITION_FIELD_NAME_FIELD);
+        parser.declareBoolean(Builder::setUseNull, USE_NULL_FIELD);
+        parser.declareField(Builder::setExcludeFrequent, p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ExcludeFrequent.forString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING);
+        parser.declareObjectArray(Builder::setRules,
+            (p, c) -> (ignoreUnknownFields ? DetectionRule.LENIENT_PARSER : DetectionRule.STRICT_PARSER).apply(p, c).build(),
+            CUSTOM_RULES_FIELD);
+        parser.declareInt(Builder::setDetectorIndex, DETECTOR_INDEX);
+
+        return parser;
     }
 
     public static final String BY = "by";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/FilterRef.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/FilterRef.java
index 7f3fb562879..9410e37250f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/FilterRef.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/FilterRef.java
@@ -14,12 +14,9 @@ import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 
 import java.io.IOException;
-import java.util.EnumMap;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 
 public class FilterRef implements ToXContentObject, Writeable {
@@ -42,28 +39,22 @@ public class FilterRef implements ToXContentObject, Writeable {
     }
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ConstructingObjectParser<FilterRef, Void> METADATA_PARSER =
-            new ConstructingObjectParser<>(FILTER_REF_FIELD.getPreferredName(), true,
-                    a -> new FilterRef((String) a[0], (FilterType) a[1]));
-    public static final ConstructingObjectParser<FilterRef, Void> CONFIG_PARSER =
-            new ConstructingObjectParser<>(FILTER_REF_FIELD.getPreferredName(), false,
-                    a -> new FilterRef((String) a[0], (FilterType) a[1]));
-    public static final Map<MlParserType, ConstructingObjectParser<FilterRef, Void>> PARSERS = new EnumMap<>(MlParserType.class);
+    public static final ConstructingObjectParser<FilterRef, Void> LENIENT_PARSER = createParser(true);
+    public static final ConstructingObjectParser<FilterRef, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ConstructingObjectParser<FilterRef, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareString(ConstructingObjectParser.constructorArg(), FILTER_ID);
-            parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
-                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                    return FilterType.fromString(p.text());
-                }
-                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-            }, FILTER_TYPE, ObjectParser.ValueType.STRING);
-        }
+    private static ConstructingObjectParser<FilterRef, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<FilterRef, Void> parser = new ConstructingObjectParser<>(FILTER_REF_FIELD.getPreferredName(),
+            ignoreUnknownFields, a -> new FilterRef((String) a[0], (FilterType) a[1]));
+
+        parser.declareString(ConstructingObjectParser.constructorArg(), FILTER_ID);
+        parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return FilterType.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, FILTER_TYPE, ObjectParser.ValueType.STRING);
+
+        return parser;
     }
 
     private final String filterId;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
index 560bac895fa..0005d16a99c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
@@ -21,7 +21,6 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
@@ -34,7 +33,6 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
-import java.util.EnumMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -85,69 +83,70 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
     public static final ParseField RESULTS_FIELD = new ParseField("jobs");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ObjectParser<Builder, Void> METADATA_PARSER = new ObjectParser<>("job_details", true, Builder::new);
-    public static final ObjectParser<Builder, Void> CONFIG_PARSER = new ObjectParser<>("job_details", false, Builder::new);
-    public static final Map<MlParserType, ObjectParser<Builder, Void>> PARSERS = new EnumMap<>(MlParserType.class);
+    public static final ObjectParser<Builder, Void> LENIENT_PARSER = createParser(true);
+    public static final ObjectParser<Builder, Void> STRICT_PARSER = createParser(false);
 
     public static final TimeValue MIN_BACKGROUND_PERSIST_INTERVAL = TimeValue.timeValueHours(1);
     public static final ByteSizeValue PROCESS_MEMORY_OVERHEAD = new ByteSizeValue(100, ByteSizeUnit.MB);
 
     public static final long DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS = 1;
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ObjectParser<Builder, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareString(Builder::setId, ID);
-            parser.declareString(Builder::setJobType, JOB_TYPE);
-            parser.declareString(Builder::setJobVersion, JOB_VERSION);
-            parser.declareStringArray(Builder::setGroups, GROUPS);
-            parser.declareStringOrNull(Builder::setDescription, DESCRIPTION);
-            parser.declareField(Builder::setCreateTime, p -> {
-                if (p.currentToken() == Token.VALUE_NUMBER) {
-                    return new Date(p.longValue());
-                } else if (p.currentToken() == Token.VALUE_STRING) {
-                    return new Date(TimeUtils.dateStringToEpoch(p.text()));
-                }
-                throw new IllegalArgumentException("unexpected token [" + p.currentToken() +
-                        "] for [" + CREATE_TIME.getPreferredName() + "]");
-            }, CREATE_TIME, ValueType.VALUE);
-            parser.declareField(Builder::setFinishedTime, p -> {
-                if (p.currentToken() == Token.VALUE_NUMBER) {
-                    return new Date(p.longValue());
-                } else if (p.currentToken() == Token.VALUE_STRING) {
-                    return new Date(TimeUtils.dateStringToEpoch(p.text()));
-                }
-                throw new IllegalArgumentException(
-                        "unexpected token [" + p.currentToken() + "] for [" + FINISHED_TIME.getPreferredName() + "]");
-            }, FINISHED_TIME, ValueType.VALUE);
-            parser.declareField(Builder::setLastDataTime, p -> {
-                if (p.currentToken() == Token.VALUE_NUMBER) {
-                    return new Date(p.longValue());
-                } else if (p.currentToken() == Token.VALUE_STRING) {
-                    return new Date(TimeUtils.dateStringToEpoch(p.text()));
-                }
-                throw new IllegalArgumentException(
-                        "unexpected token [" + p.currentToken() + "] for [" + LAST_DATA_TIME.getPreferredName() + "]");
-            }, LAST_DATA_TIME, ValueType.VALUE);
-            parser.declareLong(Builder::setEstablishedModelMemory, ESTABLISHED_MODEL_MEMORY);
-            parser.declareObject(Builder::setAnalysisConfig, AnalysisConfig.PARSERS.get(parserType), ANALYSIS_CONFIG);
-            parser.declareObject(Builder::setAnalysisLimits, AnalysisLimits.PARSERS.get(parserType), ANALYSIS_LIMITS);
-            parser.declareObject(Builder::setDataDescription, DataDescription.PARSERS.get(parserType), DATA_DESCRIPTION);
-            parser.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.PARSERS.get(parserType), MODEL_PLOT_CONFIG);
-            parser.declareLong(Builder::setRenormalizationWindowDays, RENORMALIZATION_WINDOW_DAYS);
-            parser.declareString((builder, val) -> builder.setBackgroundPersistInterval(
-                    TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
-            parser.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
-            parser.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
-            parser.declareField(Builder::setCustomSettings, (p, c) -> p.map(), CUSTOM_SETTINGS, ValueType.OBJECT);
-            parser.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID);
-            parser.declareStringOrNull(Builder::setModelSnapshotMinVersion, MODEL_SNAPSHOT_MIN_VERSION);
-            parser.declareString(Builder::setResultsIndexName, RESULTS_INDEX_NAME);
-            parser.declareBoolean(Builder::setDeleted, DELETED);
-        }
+    private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFields) {
+        ObjectParser<Builder, Void> parser = new ObjectParser<>("job_details", ignoreUnknownFields, Builder::new);
+
+        parser.declareString(Builder::setId, ID);
+        parser.declareString(Builder::setJobType, JOB_TYPE);
+        parser.declareString(Builder::setJobVersion, JOB_VERSION);
+        parser.declareStringArray(Builder::setGroups, GROUPS);
+        parser.declareStringOrNull(Builder::setDescription, DESCRIPTION);
+        parser.declareField(Builder::setCreateTime, p -> {
+            if (p.currentToken() == Token.VALUE_NUMBER) {
+                return new Date(p.longValue());
+            } else if (p.currentToken() == Token.VALUE_STRING) {
+                return new Date(TimeUtils.dateStringToEpoch(p.text()));
+            }
+            throw new IllegalArgumentException("unexpected token [" + p.currentToken() +
+                "] for [" + CREATE_TIME.getPreferredName() + "]");
+        }, CREATE_TIME, ValueType.VALUE);
+        parser.declareField(Builder::setFinishedTime, p -> {
+            if (p.currentToken() == Token.VALUE_NUMBER) {
+                return new Date(p.longValue());
+            } else if (p.currentToken() == Token.VALUE_STRING) {
+                return new Date(TimeUtils.dateStringToEpoch(p.text()));
+            }
+            throw new IllegalArgumentException(
+                "unexpected token [" + p.currentToken() + "] for [" + FINISHED_TIME.getPreferredName() + "]");
+        }, FINISHED_TIME, ValueType.VALUE);
+        parser.declareField(Builder::setLastDataTime, p -> {
+            if (p.currentToken() == Token.VALUE_NUMBER) {
+                return new Date(p.longValue());
+            } else if (p.currentToken() == Token.VALUE_STRING) {
+                return new Date(TimeUtils.dateStringToEpoch(p.text()));
+            }
+            throw new IllegalArgumentException(
+                "unexpected token [" + p.currentToken() + "] for [" + LAST_DATA_TIME.getPreferredName() + "]");
+        }, LAST_DATA_TIME, ValueType.VALUE);
+        parser.declareLong(Builder::setEstablishedModelMemory, ESTABLISHED_MODEL_MEMORY);
+        parser.declareObject(Builder::setAnalysisConfig, ignoreUnknownFields ? AnalysisConfig.LENIENT_PARSER : AnalysisConfig.STRICT_PARSER,
+            ANALYSIS_CONFIG);
+        parser.declareObject(Builder::setAnalysisLimits, ignoreUnknownFields ? AnalysisLimits.LENIENT_PARSER : AnalysisLimits.STRICT_PARSER,
+            ANALYSIS_LIMITS);
+        parser.declareObject(Builder::setDataDescription,
+            ignoreUnknownFields ? DataDescription.LENIENT_PARSER : DataDescription.STRICT_PARSER, DATA_DESCRIPTION);
+        parser.declareObject(Builder::setModelPlotConfig,
+            ignoreUnknownFields ? ModelPlotConfig.LENIENT_PARSER : ModelPlotConfig.STRICT_PARSER, MODEL_PLOT_CONFIG);
+        parser.declareLong(Builder::setRenormalizationWindowDays, RENORMALIZATION_WINDOW_DAYS);
+        parser.declareString((builder, val) -> builder.setBackgroundPersistInterval(
+            TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
+        parser.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
+        parser.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
+        parser.declareField(Builder::setCustomSettings, (p, c) -> p.map(), CUSTOM_SETTINGS, ValueType.OBJECT);
+        parser.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID);
+        parser.declareStringOrNull(Builder::setModelSnapshotMinVersion, MODEL_SNAPSHOT_MIN_VERSION);
+        parser.declareString(Builder::setResultsIndexName, RESULTS_INDEX_NAME);
+        parser.declareBoolean(Builder::setDeleted, DELETED);
+
+        return parser;
     }
 
     private final String jobId;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java
index 7b6843a2415..380f540a317 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java
@@ -44,8 +44,8 @@ public class JobUpdate implements Writeable, ToXContentObject {
             parser.declareStringArray(Builder::setGroups, Job.GROUPS);
             parser.declareStringOrNull(Builder::setDescription, Job.DESCRIPTION);
             parser.declareObjectArray(Builder::setDetectorUpdates, DetectorUpdate.PARSER, DETECTORS);
-            parser.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.CONFIG_PARSER, Job.MODEL_PLOT_CONFIG);
-            parser.declareObject(Builder::setAnalysisLimits, AnalysisLimits.CONFIG_PARSER, Job.ANALYSIS_LIMITS);
+            parser.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.STRICT_PARSER, Job.MODEL_PLOT_CONFIG);
+            parser.declareObject(Builder::setAnalysisLimits, AnalysisLimits.STRICT_PARSER, Job.ANALYSIS_LIMITS);
             parser.declareString((builder, val) -> builder.setBackgroundPersistInterval(
                     TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())), Job.BACKGROUND_PERSIST_INTERVAL);
             parser.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS);
@@ -533,7 +533,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
             PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), Detector.DETECTOR_INDEX);
             PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), Job.DESCRIPTION);
             PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (parser, parseFieldMatcher) ->
-                    DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), Detector.CUSTOM_RULES_FIELD);
+                    DetectionRule.STRICT_PARSER.apply(parser, parseFieldMatcher).build(), Detector.CUSTOM_RULES_FIELD);
         }
 
         private int detectorIndex;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfig.java
index d0995d22c7e..98aa618dd1e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfig.java
@@ -12,11 +12,8 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 
 import java.io.IOException;
-import java.util.EnumMap;
-import java.util.Map;
 import java.util.Objects;
 
 public class ModelPlotConfig implements ToXContentObject, Writeable {
@@ -26,24 +23,17 @@ public class ModelPlotConfig implements ToXContentObject, Writeable {
     public static final ParseField TERMS_FIELD = new ParseField("terms");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ConstructingObjectParser<ModelPlotConfig, Void> METADATA_PARSER =
-            new ConstructingObjectParser<>(TYPE_FIELD.getPreferredName(), true,
-                    a -> new ModelPlotConfig((boolean) a[0], (String) a[1]));
-    public static final ConstructingObjectParser<ModelPlotConfig, Void> CONFIG_PARSER =
-            new ConstructingObjectParser<>(TYPE_FIELD.getPreferredName(), false,
-                    a -> new ModelPlotConfig((boolean) a[0], (String) a[1]));
-    public static final Map<MlParserType, ConstructingObjectParser<ModelPlotConfig, Void>> PARSERS =
-            new EnumMap<>(MlParserType.class);
+    public static final ConstructingObjectParser<ModelPlotConfig, Void> LENIENT_PARSER = createParser(true);
+    public static final ConstructingObjectParser<ModelPlotConfig, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ConstructingObjectParser<ModelPlotConfig, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD);
-            parser.declareString(ConstructingObjectParser.optionalConstructorArg(), TERMS_FIELD);
-        }
+    private static ConstructingObjectParser<ModelPlotConfig, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<ModelPlotConfig, Void> parser = new ConstructingObjectParser<>(TYPE_FIELD.getPreferredName(),
+            ignoreUnknownFields, a -> new ModelPlotConfig((boolean) a[0], (String) a[1]));
+
+        parser.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD);
+        parser.declareString(ConstructingObjectParser.optionalConstructorArg(), TERMS_FIELD);
+
+        return parser;
     }
 
     private final boolean enabled;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleCondition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleCondition.java
index 378ceaca6c4..25fac755931 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleCondition.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleCondition.java
@@ -14,12 +14,9 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 
 import java.io.IOException;
-import java.util.EnumMap;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 
 public class RuleCondition implements ToXContentObject, Writeable {
@@ -30,35 +27,28 @@ public class RuleCondition implements ToXContentObject, Writeable {
     public static final ParseField VALUE_FIELD = new ParseField("value");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
-    public static final ConstructingObjectParser<RuleCondition, Void> METADATA_PARSER =
-            new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(), true,
-                    a -> new RuleCondition((AppliesTo) a[0], (Operator) a[1], (double) a[2]));
-    public static final ConstructingObjectParser<RuleCondition, Void> CONFIG_PARSER =
-            new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(), false,
-                    a -> new RuleCondition((AppliesTo) a[0], (Operator) a[1], (double) a[2]));
-    public static final Map<MlParserType, ConstructingObjectParser<RuleCondition, Void>> PARSERS =
-            new EnumMap<>(MlParserType.class);
+    public static final ConstructingObjectParser<RuleCondition, Void> LENIENT_PARSER = createParser(true);
+    public static final ConstructingObjectParser<RuleCondition, Void> STRICT_PARSER = createParser(false);
 
-    static {
-        PARSERS.put(MlParserType.METADATA, METADATA_PARSER);
-        PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER);
-        for (MlParserType parserType : MlParserType.values()) {
-            ConstructingObjectParser<RuleCondition, Void> parser = PARSERS.get(parserType);
-            assert parser != null;
-            parser.declareField(ConstructingObjectParser.constructorArg(), p -> {
-                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                    return AppliesTo.fromString(p.text());
-                }
-                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-            }, APPLIES_TO_FIELD, ValueType.STRING);
-            parser.declareField(ConstructingObjectParser.constructorArg(), p -> {
-                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
-                    return Operator.fromString(p.text());
-                }
-                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
-            }, Operator.OPERATOR_FIELD, ValueType.STRING);
-            parser.declareDouble(ConstructingObjectParser.constructorArg(), VALUE_FIELD);
-        }
+    private static ConstructingObjectParser<RuleCondition, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<RuleCondition, Void> parser = new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(),
+            ignoreUnknownFields, a -> new RuleCondition((AppliesTo) a[0], (Operator) a[1], (double) a[2]));
+
+        parser.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return AppliesTo.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, APPLIES_TO_FIELD, ValueType.STRING);
+        parser.declareField(ConstructingObjectParser.constructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return Operator.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, Operator.OPERATOR_FIELD, ValueType.STRING);
+        parser.declareDouble(ConstructingObjectParser.constructorArg(), VALUE_FIELD);
+
+        return parser;
     }
 
     private final AppliesTo appliesTo;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java
index 0b11fa0e15b..0c633c6ead2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java
@@ -17,7 +17,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
@@ -32,13 +31,14 @@ import java.util.stream.Collectors;
 
 public class RuleScope implements ToXContentObject, Writeable {
 
-    public static ContextParser<Void, RuleScope> parser(MlParserType parserType) {
+    public static ContextParser<Void, RuleScope> parser(boolean ignoreUnknownFields) {
         return (p, c) -> {
             Map<String, Object> unparsedScope = p.map();
             if (unparsedScope.isEmpty()) {
                 return new RuleScope();
             }
-            ConstructingObjectParser<FilterRef, Void> filterRefParser = FilterRef.PARSERS.get(parserType);
+            ConstructingObjectParser<FilterRef, Void> filterRefParser =
+                ignoreUnknownFields ? FilterRef.LENIENT_PARSER : FilterRef.STRICT_PARSER;
             Map<String, FilterRef> scope = new HashMap<>();
             for (Map.Entry<String, Object> entry : unparsedScope.entrySet()) {
                 try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfigTests.java
index ef89200b765..f91d7389366 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfigTests.java
@@ -32,7 +32,7 @@ public class ChunkingConfigTests extends AbstractSerializingTestCase<ChunkingCon
 
     @Override
     protected ChunkingConfig doParseInstance(XContentParser parser) {
-        return ChunkingConfig.CONFIG_PARSER.apply(parser, null);
+        return ChunkingConfig.STRICT_PARSER.apply(parser, null);
     }
 
     public void testConstructorGivenAutoAndTimeSpan() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
index d59ef16dfdf..ffc13655d22 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
@@ -140,7 +140,7 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
 
     @Override
     protected DatafeedConfig doParseInstance(XContentParser parser) {
-        return DatafeedConfig.CONFIG_PARSER.apply(parser, null).build();
+        return DatafeedConfig.STRICT_PARSER.apply(parser, null).build();
     }
 
     private static final String FUTURE_DATAFEED = "{\n" +
@@ -156,7 +156,7 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_DATAFEED);
         XContentParseException e = expectThrows(XContentParseException.class,
-                () -> DatafeedConfig.CONFIG_PARSER.apply(parser, null).build());
+                () -> DatafeedConfig.STRICT_PARSER.apply(parser, null).build());
         assertEquals("[6:5] [datafeed_config] unknown field [tomorrows_technology_today], parser not found", e.getMessage());
     }
 
@@ -164,7 +164,7 @@ public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedCon
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_DATAFEED);
         // Unlike the config version of this test, the metadata parser should tolerate the unknown future field
-        assertNotNull(DatafeedConfig.METADATA_PARSER.apply(parser, null).build());
+        assertNotNull(DatafeedConfig.LENIENT_PARSER.apply(parser, null).build());
     }
 
     public void testCopyConstructor() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java
index 6c54eb78189..00a1586c1d5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java
@@ -120,7 +120,7 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase<AnalysisCon
 
     @Override
     protected AnalysisConfig doParseInstance(XContentParser parser) {
-        return AnalysisConfig.CONFIG_PARSER.apply(parser, null).build();
+        return AnalysisConfig.STRICT_PARSER.apply(parser, null).build();
     }
 
     public void testFieldConfiguration_singleDetector_notPreSummarised() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimitsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimitsTests.java
index 39620ab1c7d..f73fcc41bce 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimitsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimitsTests.java
@@ -41,14 +41,14 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
 
     @Override
     protected AnalysisLimits doParseInstance(XContentParser parser) {
-        return AnalysisLimits.CONFIG_PARSER.apply(parser, null);
+        return AnalysisLimits.STRICT_PARSER.apply(parser, null);
     }
 
     public void testParseModelMemoryLimitGivenNegativeNumber() throws IOException {
         String json = "{\"model_memory_limit\": -1}";
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
-        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.CONFIG_PARSER.apply(parser, null));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.STRICT_PARSER.apply(parser, null));
         assertThat(ExceptionsHelper.detailedMessage(e), containsString("model_memory_limit must be at least 1 MiB. Value = -1"));
     }
 
@@ -56,7 +56,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
         String json = "{\"model_memory_limit\": 0}";
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
-        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.CONFIG_PARSER.apply(parser, null));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.STRICT_PARSER.apply(parser, null));
         assertThat(ExceptionsHelper.detailedMessage(e), containsString("model_memory_limit must be at least 1 MiB. Value = 0"));
     }
 
@@ -65,7 +65,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
 
-        AnalysisLimits limits = AnalysisLimits.CONFIG_PARSER.apply(parser, null);
+        AnalysisLimits limits = AnalysisLimits.STRICT_PARSER.apply(parser, null);
 
         assertThat(limits.getModelMemoryLimit(), equalTo(2048L));
     }
@@ -74,7 +74,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
         String json = "{\"model_memory_limit\":\"-4MB\"}";
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
-        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.CONFIG_PARSER.apply(parser, null));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.STRICT_PARSER.apply(parser, null));
         assertThat(ExceptionsHelper.detailedMessage(e), containsString("Values less than -1 bytes are not supported: -4mb"));
     }
 
@@ -82,7 +82,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
         String json = "{\"model_memory_limit\":\"0MB\"}";
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
-        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.CONFIG_PARSER.apply(parser, null));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.STRICT_PARSER.apply(parser, null));
         assertThat(ExceptionsHelper.detailedMessage(e), containsString("model_memory_limit must be at least 1 MiB. Value = 0"));
     }
 
@@ -90,7 +90,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
         String json = "{\"model_memory_limit\":\"1000Kb\"}";
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
-        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.CONFIG_PARSER.apply(parser, null));
+        XContentParseException e = expectThrows(XContentParseException.class, () -> AnalysisLimits.STRICT_PARSER.apply(parser, null));
         assertThat(ExceptionsHelper.detailedMessage(e), containsString("model_memory_limit must be at least 1 MiB. Value = 0"));
     }
 
@@ -99,7 +99,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
 
-        AnalysisLimits limits = AnalysisLimits.CONFIG_PARSER.apply(parser, null);
+        AnalysisLimits limits = AnalysisLimits.STRICT_PARSER.apply(parser, null);
 
         assertThat(limits.getModelMemoryLimit(), equalTo(4096L));
     }
@@ -109,7 +109,7 @@ public class AnalysisLimitsTests extends AbstractSerializingTestCase<AnalysisLim
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
 
-        AnalysisLimits limits = AnalysisLimits.CONFIG_PARSER.apply(parser, null);
+        AnalysisLimits limits = AnalysisLimits.STRICT_PARSER.apply(parser, null);
 
         assertThat(limits.getModelMemoryLimit(), equalTo(1L));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java
index bb7c329cf45..013e7197024 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java
@@ -212,7 +212,7 @@ public class DataDescriptionTests extends AbstractSerializingTestCase<DataDescri
         XContentParser parser = JsonXContent.jsonXContent
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json.streamInput());
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> DataDescription.CONFIG_PARSER.apply(parser, null));
+                () -> DataDescription.STRICT_PARSER.apply(parser, null));
         assertThat(ex.getMessage(), containsString("[data_description] failed to parse field [format]"));
         Throwable cause = ex.getCause();
         assertNotNull(cause);
@@ -226,7 +226,7 @@ public class DataDescriptionTests extends AbstractSerializingTestCase<DataDescri
         XContentParser parser = JsonXContent.jsonXContent
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json.streamInput());
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> DataDescription.CONFIG_PARSER.apply(parser, null));
+                () -> DataDescription.STRICT_PARSER.apply(parser, null));
         assertThat(ex.getMessage(), containsString("[data_description] failed to parse field [field_delimiter]"));
         Throwable cause = ex.getCause();
         assertNotNull(cause);
@@ -240,7 +240,7 @@ public class DataDescriptionTests extends AbstractSerializingTestCase<DataDescri
         XContentParser parser = JsonXContent.jsonXContent
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json.streamInput());
         XContentParseException ex = expectThrows(XContentParseException.class,
-                () -> DataDescription.CONFIG_PARSER.apply(parser, null));
+                () -> DataDescription.STRICT_PARSER.apply(parser, null));
         assertThat(ex.getMessage(), containsString("[data_description] failed to parse field [quote_character]"));
         Throwable cause = ex.getCause();
         assertNotNull(cause);
@@ -284,7 +284,7 @@ public class DataDescriptionTests extends AbstractSerializingTestCase<DataDescri
 
     @Override
     protected DataDescription doParseInstance(XContentParser parser) {
-        return DataDescription.CONFIG_PARSER.apply(parser, null).build();
+        return DataDescription.STRICT_PARSER.apply(parser, null).build();
     }
 
     protected DataDescription mutateInstance(DataDescription instance) throws java.io.IOException {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java
index a57bd24eda0..4615b788373 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java
@@ -90,7 +90,7 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
 
     @Override
     protected DetectionRule doParseInstance(XContentParser parser) {
-        return DetectionRule.CONFIG_PARSER.apply(parser, null).build();
+        return DetectionRule.STRICT_PARSER.apply(parser, null).build();
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
index c9be0d3b892..762d1d7ec39 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
@@ -182,7 +182,7 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
 
     @Override
     protected Detector doParseInstance(XContentParser parser) {
-        return Detector.CONFIG_PARSER.apply(parser, null).build();
+        return Detector.STRICT_PARSER.apply(parser, null).build();
     }
 
     public void testVerifyFieldNames_givenInvalidChars() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java
index 241bf659332..71530110687 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java
@@ -20,11 +20,11 @@ public class FilterRefTests extends AbstractSerializingTestCase<FilterRef> {
 
     @Override
     protected FilterRef doParseInstance(XContentParser parser) throws IOException {
-        return FilterRef.CONFIG_PARSER.parse(parser, null);
+        return FilterRef.STRICT_PARSER.parse(parser, null);
     }
 
     @Override
     protected Writeable.Reader<FilterRef> instanceReader() {
         return FilterRef::new;
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java
index 0f35abd4bcf..88d9b07816d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java
@@ -74,14 +74,14 @@ public class JobTests extends AbstractSerializingTestCase<Job> {
 
     @Override
     protected Job doParseInstance(XContentParser parser) {
-        return Job.CONFIG_PARSER.apply(parser, null).build();
+        return Job.STRICT_PARSER.apply(parser, null).build();
     }
 
     public void testFutureConfigParse() throws IOException {
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_JOB);
         XContentParseException e = expectThrows(XContentParseException.class,
-                () -> Job.CONFIG_PARSER.apply(parser, null).build());
+                () -> Job.STRICT_PARSER.apply(parser, null).build());
         assertEquals("[4:5] [job_details] unknown field [tomorrows_technology_today], parser not found", e.getMessage());
     }
 
@@ -89,7 +89,7 @@ public class JobTests extends AbstractSerializingTestCase<Job> {
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_JOB);
         // Unlike the config version of this test, the metadata parser should tolerate the unknown future field
-        assertNotNull(Job.METADATA_PARSER.apply(parser, null).build());
+        assertNotNull(Job.LENIENT_PARSER.apply(parser, null).build());
     }
 
     public void testConstructor_GivenEmptyJobConfiguration() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfigTests.java
index aa54a174194..c57f637d572 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfigTests.java
@@ -31,6 +31,6 @@ public class ModelPlotConfigTests extends AbstractSerializingTestCase<ModelPlotC
 
     @Override
     protected ModelPlotConfig doParseInstance(XContentParser parser) {
-        return ModelPlotConfig.CONFIG_PARSER.apply(parser, null);
+        return ModelPlotConfig.STRICT_PARSER.apply(parser, null);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java
index 07122818d55..636b4621187 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java
@@ -29,7 +29,7 @@ public class RuleConditionTests extends AbstractSerializingTestCase<RuleConditio
 
     @Override
     protected RuleCondition doParseInstance(XContentParser parser) {
-        return RuleCondition.CONFIG_PARSER.apply(parser, null);
+        return RuleCondition.STRICT_PARSER.apply(parser, null);
     }
 
     public void testEqualsGivenSameObject() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
index fd35c4b3bad..e16ac2f9970 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
@@ -79,7 +79,7 @@ public class MlMetadataTests extends AbstractSerializingTestCase<MlMetadata> {
 
     @Override
     protected MlMetadata doParseInstance(XContentParser parser) {
-        return MlMetadata.METADATA_PARSER.apply(parser, null).build();
+        return MlMetadata.LENIENT_PARSER.apply(parser, null).build();
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java
index 2fe2c0b334c..64110774535 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/CategorizationAnalyzerConfigTests.java
@@ -9,7 +9,6 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
-import org.elasticsearch.xpack.core.ml.MlParserType;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -72,7 +71,7 @@ public class CategorizationAnalyzerConfigTests extends AbstractSerializingTestCa
 
     @Override
     protected CategorizationAnalyzerConfig doParseInstance(XContentParser parser) throws IOException {
-        return CategorizationAnalyzerConfig.buildFromXContentObject(parser, MlParserType.CONFIG);
+        return CategorizationAnalyzerConfig.buildFromXContentObject(parser, false);
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java
index e5f1c32fb8c..807e94c2d90 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java
@@ -86,6 +86,6 @@ public class JobBuilderTests extends AbstractSerializingTestCase<Job.Builder> {
 
     @Override
     protected Job.Builder doParseInstance(XContentParser parser) {
-        return Job.CONFIG_PARSER.apply(parser, null);
+        return Job.STRICT_PARSER.apply(parser, null);
     }
 }

From 3b050194f9f014735dea82bd96fc63efb43d5cc7 Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Thu, 26 Jul 2018 09:00:35 -0700
Subject: [PATCH 194/260] Docs: Fix callouts in put license HL REST docs
 (#32363)

---
 docs/java-rest/high-level/licensing/put-license.asciidoc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/java-rest/high-level/licensing/put-license.asciidoc b/docs/java-rest/high-level/licensing/put-license.asciidoc
index 7456fdd3c02..a270d658ddd 100644
--- a/docs/java-rest/high-level/licensing/put-license.asciidoc
+++ b/docs/java-rest/high-level/licensing/put-license.asciidoc
@@ -33,10 +33,9 @@ include-tagged::{doc-tests}/LicensingDocumentationIT.java[put-license-response]
 --------------------------------------------------
 <1> The status of the license
 <2> Make sure that the license is valid.
-<3> Check the acknowledge flag.
-<4> It should be true if license is acknowledge.
-<5> Otherwise we can see the acknowledge messages in `acknowledgeHeader()` and check
-component-specific messages in `acknowledgeMessages()`.
+<3> Check the acknowledge flag. It should be true if license is acknowledged.
+<4> Otherwise we can see the acknowledge messages in `acknowledgeHeader()`
+<5> and check component-specific messages in `acknowledgeMessages()`.
 
 [[java-rest-high-put-license-async]]
 ==== Asynchronous Execution

From df579f8bce005601d8b6c24f91699e67a92eac19 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 26 Jul 2018 09:02:06 -0700
Subject: [PATCH 195/260] Painless: Clean Up PainlessClass Variables (#32380)

Removes the variables name, clazz, and type as they are unnecessary. Renames
staticMembers -> staticFields, members -> fields, getters -> getterMethodHandles, and
setters -> setterMethodHandles.
---
 .../painless/spi/PainlessExtension.java       |  4 +-
 .../java/org/elasticsearch/painless/Def.java  |  8 +--
 .../painless/PainlessExplainError.java        |  3 +-
 .../painless/lookup/PainlessClass.java        | 35 ++++------
 .../painless/lookup/PainlessClassBuilder.java | 35 +++-------
 .../painless/lookup/PainlessLookup.java       |  4 +-
 .../lookup/PainlessLookupBuilder.java         | 69 ++++++++++---------
 .../elasticsearch/painless/node/ENewObj.java  |  8 ++-
 .../elasticsearch/painless/node/PBrace.java   |  4 +-
 .../painless/node/PCallInvoke.java            |  4 +-
 .../elasticsearch/painless/node/PField.java   |  6 +-
 .../painless/node/PSubListShortcut.java       | 15 ++--
 .../painless/node/PSubMapShortcut.java        | 15 ++--
 .../painless/PainlessDocGenerator.java        | 49 +++++++------
 .../painless/node/NodeToStringTests.java      | 14 ++--
 15 files changed, 131 insertions(+), 142 deletions(-)

diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessExtension.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessExtension.java
index 9434e6986c0..eb971353437 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessExtension.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/PainlessExtension.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.painless.spi;
 
+import org.elasticsearch.script.ScriptContext;
+
 import java.util.List;
 import java.util.Map;
 
-import org.elasticsearch.script.ScriptContext;
-
 public interface PainlessExtension {
 
     Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index dad8da06e76..8a90f53b4fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -421,7 +421,7 @@ public final class Def {
             PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz);
 
             if (struct != null) {
-                MethodHandle handle = struct.getters.get(name);
+                MethodHandle handle = struct.getterMethodHandles.get(name);
                 if (handle != null) {
                     return handle;
                 }
@@ -431,7 +431,7 @@ public final class Def {
                 struct = painlessLookup.getPainlessStructFromJavaClass(iface);
 
                 if (struct != null) {
-                    MethodHandle handle = struct.getters.get(name);
+                    MethodHandle handle = struct.getterMethodHandles.get(name);
                     if (handle != null) {
                         return handle;
                     }
@@ -492,7 +492,7 @@ public final class Def {
             PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz);
 
             if (struct != null) {
-                MethodHandle handle = struct.setters.get(name);
+                MethodHandle handle = struct.setterMethodHandles.get(name);
                 if (handle != null) {
                     return handle;
                 }
@@ -502,7 +502,7 @@ public final class Def {
                 struct = painlessLookup.getPainlessStructFromJavaClass(iface);
 
                 if (struct != null) {
-                    MethodHandle handle = struct.setters.get(name);
+                    MethodHandle handle = struct.setterMethodHandles.get(name);
                     if (handle != null) {
                         return handle;
                     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
index 1236c4977e8..7bef028c7d1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless;
 import org.elasticsearch.painless.api.Debug;
 import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessLookup;
+import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.script.ScriptException;
 
 import java.util.List;
@@ -58,7 +59,7 @@ public class PainlessExplainError extends Error {
             javaClassName = objectToExplain.getClass().getName();
             PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(objectToExplain.getClass());
             if (struct != null) {
-                painlessClassName = struct.name;
+                painlessClassName = PainlessLookupUtility.typeToCanonicalTypeName(objectToExplain.getClass());
             }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
index 57b18bc60da..24dcf0ebdba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
@@ -19,47 +19,38 @@
 
 package org.elasticsearch.painless.lookup;
 
-import org.objectweb.asm.Type;
-
 import java.lang.invoke.MethodHandle;
 import java.util.Collections;
 import java.util.Map;
 
 public final class PainlessClass {
-    public final String name;
-    public final Class<?> clazz;
-    public final Type type;
-
     public final Map<String, PainlessMethod> constructors;
     public final Map<String, PainlessMethod> staticMethods;
     public final Map<String, PainlessMethod> methods;
 
-    public final Map<String, PainlessField> staticMembers;
-    public final Map<String, PainlessField> members;
+    public final Map<String, PainlessField> staticFields;
+    public final Map<String, PainlessField> fields;
 
-    public final Map<String, MethodHandle> getters;
-    public final Map<String, MethodHandle> setters;
+    public final Map<String, MethodHandle> getterMethodHandles;
+    public final Map<String, MethodHandle> setterMethodHandles;
 
     public final PainlessMethod functionalMethod;
 
-    PainlessClass(String name, Class<?> clazz, Type type,
-                  Map<String, PainlessMethod> constructors, Map<String, PainlessMethod> staticMethods, Map<String, PainlessMethod> methods,
-                  Map<String, PainlessField> staticMembers, Map<String, PainlessField> members,
-                  Map<String, MethodHandle> getters, Map<String, MethodHandle> setters,
-                  PainlessMethod functionalMethod) {
-        this.name = name;
-        this.clazz = clazz;
-        this.type = type;
+    PainlessClass(Map<String, PainlessMethod> constructors,
+            Map<String, PainlessMethod> staticMethods, Map<String, PainlessMethod> methods,
+            Map<String, PainlessField> staticFields, Map<String, PainlessField> fields,
+            Map<String, MethodHandle> getterMethodHandles, Map<String, MethodHandle> setterMethodHandles,
+            PainlessMethod functionalMethod) {
 
         this.constructors = Collections.unmodifiableMap(constructors);
         this.staticMethods = Collections.unmodifiableMap(staticMethods);
         this.methods = Collections.unmodifiableMap(methods);
 
-        this.staticMembers = Collections.unmodifiableMap(staticMembers);
-        this.members = Collections.unmodifiableMap(members);
+        this.staticFields = Collections.unmodifiableMap(staticFields);
+        this.fields = Collections.unmodifiableMap(fields);
 
-        this.getters = Collections.unmodifiableMap(getters);
-        this.setters = Collections.unmodifiableMap(setters);
+        this.getterMethodHandles = Collections.unmodifiableMap(getterMethodHandles);
+        this.setterMethodHandles = Collections.unmodifiableMap(setterMethodHandles);
 
         this.functionalMethod = functionalMethod;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
index 0eda3660f0b..2f41ed5dca8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
@@ -19,52 +19,39 @@
 
 package org.elasticsearch.painless.lookup;
 
-import org.objectweb.asm.Type;
-
 import java.lang.invoke.MethodHandle;
 import java.util.HashMap;
 import java.util.Map;
 
 final class PainlessClassBuilder {
-    final String name;
-    final Class<?> clazz;
-    final Type type;
-
     final Map<String, PainlessMethod> constructors;
     final Map<String, PainlessMethod> staticMethods;
     final Map<String, PainlessMethod> methods;
 
-    final Map<String, PainlessField> staticMembers;
-    final Map<String, PainlessField> members;
+    final Map<String, PainlessField> staticFields;
+    final Map<String, PainlessField> fields;
 
-    final Map<String, MethodHandle> getters;
-    final Map<String, MethodHandle> setters;
+    final Map<String, MethodHandle> getterMethodHandles;
+    final Map<String, MethodHandle> setterMethodHandles;
 
     PainlessMethod functionalMethod;
 
-    PainlessClassBuilder(String name, Class<?> clazz, Type type) {
-        this.name = name;
-        this.clazz = clazz;
-        this.type = type;
-
+    PainlessClassBuilder() {
         constructors = new HashMap<>();
         staticMethods = new HashMap<>();
         methods = new HashMap<>();
 
-        staticMembers = new HashMap<>();
-        members = new HashMap<>();
+        staticFields = new HashMap<>();
+        fields = new HashMap<>();
 
-        getters = new HashMap<>();
-        setters = new HashMap<>();
+        getterMethodHandles = new HashMap<>();
+        setterMethodHandles = new HashMap<>();
 
         functionalMethod = null;
     }
 
     PainlessClass build() {
-        return new PainlessClass(name, clazz, type,
-            constructors, staticMethods, methods,
-            staticMembers, members,
-            getters, setters,
-            functionalMethod);
+        return new PainlessClass(constructors, staticMethods, methods, staticFields, fields,
+                getterMethodHandles, setterMethodHandles, functionalMethod);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
index bcecd7bbdc7..67c04498a58 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java
@@ -29,8 +29,8 @@ import java.util.Map;
  */
 public final class PainlessLookup {
 
-    public Collection<PainlessClass> getStructs() {
-        return classesToPainlessClasses.values();
+    public Collection<Class<?>> getStructs() {
+        return classesToPainlessClasses.keySet();
     }
 
     private final Map<String, Class<?>> canonicalClassNamesToClasses;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 519227bb901..3675cc7cd0f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -179,8 +179,7 @@ public class PainlessLookupBuilder {
         classesToPainlessClassBuilders = new HashMap<>();
 
         canonicalClassNamesToClasses.put(DEF_CLASS_NAME, def.class);
-        classesToPainlessClassBuilders.put(def.class,
-                new PainlessClassBuilder(DEF_CLASS_NAME, Object.class, org.objectweb.asm.Type.getType(Object.class)));
+        classesToPainlessClassBuilders.put(def.class, new PainlessClassBuilder());
     }
 
     private Class<?> canonicalTypeNameToType(String canonicalTypeName) {
@@ -234,17 +233,21 @@ public class PainlessLookupBuilder {
             throw new IllegalArgumentException("invalid class name [" + canonicalClassName + "]");
         }
 
+
+        Class<?> existingClass = canonicalClassNamesToClasses.get(typeToCanonicalTypeName(clazz));
+
+        if (existingClass != null && existingClass != clazz) {
+            throw new IllegalArgumentException("class [" + canonicalClassName + "] " +
+                    "cannot represent multiple java classes with the same name from different class loaders");
+        }
+
         PainlessClassBuilder existingPainlessClassBuilder = classesToPainlessClassBuilders.get(clazz);
 
         if (existingPainlessClassBuilder == null) {
-            PainlessClassBuilder painlessClassBuilder =
-                    new PainlessClassBuilder(canonicalClassName, clazz, org.objectweb.asm.Type.getType(clazz));
+            PainlessClassBuilder painlessClassBuilder = new PainlessClassBuilder();
 
             canonicalClassNamesToClasses.put(canonicalClassName, clazz);
             classesToPainlessClassBuilders.put(clazz, painlessClassBuilder);
-        } else if (existingPainlessClassBuilder.clazz.equals(clazz) == false) {
-            throw new IllegalArgumentException("class [" + canonicalClassName + "] " +
-                    "cannot represent multiple java classes with the same name from different class loaders");
         }
 
         String javaClassName = clazz.getName();
@@ -265,7 +268,7 @@ public class PainlessLookupBuilder {
 
                     canonicalClassNamesToClasses.put(importedCanonicalClassName, clazz);
                 }
-            } else if (importedPainlessClass.equals(clazz) == false) {
+            } else if (importedPainlessClass != clazz) {
                 throw new IllegalArgumentException("imported class [" + importedCanonicalClassName + "] cannot represent multiple " +
                         "classes [" + canonicalClassName + "] and [" + typeToCanonicalTypeName(importedPainlessClass) + "]");
             } else if (importClassName == false) {
@@ -504,10 +507,10 @@ public class PainlessLookupBuilder {
 
             if (painlessMethod == null) {
                 org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
-                MethodHandle javaMethodHandle;
+                MethodHandle methodHandle;
 
                 try {
-                    javaMethodHandle = MethodHandles.publicLookup().in(targetClass).unreflect(javaMethod);
+                    methodHandle = MethodHandles.publicLookup().in(targetClass).unreflect(javaMethod);
                 } catch (IllegalAccessException iae) {
                     throw new IllegalArgumentException("static method handle [[" + targetClass.getCanonicalName() + "], " +
                             "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
@@ -516,7 +519,7 @@ public class PainlessLookupBuilder {
                 painlessMethod = painlessMethodCache.computeIfAbsent(
                         new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
                         key -> new PainlessMethod(methodName, targetClass, null, returnType,
-                                typeParameters, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
+                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle));
 
                 painlessClassBuilder.staticMethods.put(painlessMethodKey, painlessMethod);
             } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
@@ -535,18 +538,18 @@ public class PainlessLookupBuilder {
 
             if (painlessMethod == null) {
                 org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
-                MethodHandle javaMethodHandle;
+                MethodHandle methodHandle;
 
                 if (augmentedClass == null) {
                     try {
-                        javaMethodHandle = MethodHandles.publicLookup().in(targetClass).unreflect(javaMethod);
+                        methodHandle = MethodHandles.publicLookup().in(targetClass).unreflect(javaMethod);
                     } catch (IllegalAccessException iae) {
                         throw new IllegalArgumentException("method handle [[" + targetClass.getCanonicalName() + "], " +
                                 "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
                     }
                 } else {
                     try {
-                        javaMethodHandle = MethodHandles.publicLookup().in(augmentedClass).unreflect(javaMethod);
+                        methodHandle = MethodHandles.publicLookup().in(augmentedClass).unreflect(javaMethod);
                     } catch (IllegalAccessException iae) {
                         throw new IllegalArgumentException("method handle [[" + targetClass.getCanonicalName() + "], " +
                                 "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found " +
@@ -557,7 +560,7 @@ public class PainlessLookupBuilder {
                 painlessMethod = painlessMethodCache.computeIfAbsent(
                         new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
                         key -> new PainlessMethod(methodName, targetClass, augmentedClass, returnType,
-                                typeParameters, asmMethod, javaMethod.getModifiers(), javaMethodHandle));
+                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle));
 
                 painlessClassBuilder.methods.put(painlessMethodKey, painlessMethod);
             } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
@@ -650,7 +653,7 @@ public class PainlessLookupBuilder {
                 throw new IllegalArgumentException("static field [[" + targetCanonicalClassName + "]. [" + fieldName + "]] must be final");
             }
 
-            PainlessField painlessField = painlessClassBuilder.staticMembers.get(painlessFieldKey);
+            PainlessField painlessField = painlessClassBuilder.staticFields.get(painlessFieldKey);
 
             if (painlessField == null) {
                 painlessField = painlessFieldCache.computeIfAbsent(
@@ -658,7 +661,7 @@ public class PainlessLookupBuilder {
                         key -> new PainlessField(fieldName, javaField.getName(), targetClass,
                                 typeParameter, javaField.getModifiers(), null, null));
 
-                painlessClassBuilder.staticMembers.put(painlessFieldKey, painlessField);
+                painlessClassBuilder.staticFields.put(painlessFieldKey, painlessField);
             } else if (painlessField.clazz != typeParameter) {
                 throw new IllegalArgumentException("cannot have static fields " +
                         "[[" + targetCanonicalClassName + "], [" + fieldName + "], [" +
@@ -674,7 +677,7 @@ public class PainlessLookupBuilder {
                 methodHandleGetter = MethodHandles.publicLookup().unreflectGetter(javaField);
             } catch (IllegalAccessException iae) {
                 throw new IllegalArgumentException(
-                        "method handle getter not found for field [[" + targetCanonicalClassName + "], [" + fieldName + "]]");
+                        "getter method handle not found for field [[" + targetCanonicalClassName + "], [" + fieldName + "]]");
             }
 
             MethodHandle methodHandleSetter;
@@ -683,10 +686,10 @@ public class PainlessLookupBuilder {
                 methodHandleSetter = MethodHandles.publicLookup().unreflectSetter(javaField);
             } catch (IllegalAccessException iae) {
                 throw new IllegalArgumentException(
-                        "method handle setter not found for field [[" + targetCanonicalClassName + "], [" + fieldName + "]]");
+                        "setter method handle not found for field [[" + targetCanonicalClassName + "], [" + fieldName + "]]");
             }
 
-            PainlessField painlessField = painlessClassBuilder.members.get(painlessFieldKey);
+            PainlessField painlessField = painlessClassBuilder.fields.get(painlessFieldKey);
 
             if (painlessField == null) {
                 painlessField = painlessFieldCache.computeIfAbsent(
@@ -694,7 +697,7 @@ public class PainlessLookupBuilder {
                         key -> new PainlessField(fieldName, javaField.getName(), targetClass,
                                 typeParameter, javaField.getModifiers(), methodHandleGetter, methodHandleSetter));
 
-                painlessClassBuilder.members.put(fieldName, painlessField);
+                painlessClassBuilder.fields.put(fieldName, painlessField);
             } else if (painlessField.clazz != typeParameter) {
                 throw new IllegalArgumentException("cannot have fields " +
                         "[[" + targetCanonicalClassName + "], [" + fieldName + "], [" +
@@ -771,14 +774,14 @@ public class PainlessLookupBuilder {
             }
         }
 
-        for (Map.Entry<String, PainlessField> painlessFieldEntry : originalPainlessClassBuilder.members.entrySet()) {
+        for (Map.Entry<String, PainlessField> painlessFieldEntry : originalPainlessClassBuilder.fields.entrySet()) {
             String painlessFieldKey = painlessFieldEntry.getKey();
             PainlessField newPainlessField = painlessFieldEntry.getValue();
-            PainlessField existingPainlessField = targetPainlessClassBuilder.members.get(painlessFieldKey);
+            PainlessField existingPainlessField = targetPainlessClassBuilder.fields.get(painlessFieldKey);
 
             if (existingPainlessField == null || existingPainlessField.target != newPainlessField.target &&
                     existingPainlessField.target.isAssignableFrom(newPainlessField.target)) {
-                targetPainlessClassBuilder.members.put(painlessFieldKey, newPainlessField);
+                targetPainlessClassBuilder.fields.put(painlessFieldKey, newPainlessField);
             }
         }
     }
@@ -796,34 +799,32 @@ public class PainlessLookupBuilder {
 
             if (typeParametersSize == 0 && methodName.startsWith("get") && methodName.length() > 3 &&
                     Character.isUpperCase(methodName.charAt(3))) {
-                painlessClassBuilder.getters.putIfAbsent(
+                painlessClassBuilder.getterMethodHandles.putIfAbsent(
                         Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.handle);
             } else if (typeParametersSize == 0 && methodName.startsWith("is") && methodName.length() > 2 &&
                     Character.isUpperCase(methodName.charAt(2))) {
-                painlessClassBuilder.getters.putIfAbsent(
+                painlessClassBuilder.getterMethodHandles.putIfAbsent(
                         Character.toLowerCase(methodName.charAt(2)) + methodName.substring(3), painlessMethod.handle);
             } else if (typeParametersSize == 1 && methodName.startsWith("set") && methodName.length() > 3 &&
                     Character.isUpperCase(methodName.charAt(3))) {
-                painlessClassBuilder.setters.putIfAbsent(
+                painlessClassBuilder.setterMethodHandles.putIfAbsent(
                         Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.handle);
             }
         }
 
-        for (PainlessField painlessField : painlessClassBuilder.members.values()) {
-            painlessClassBuilder.getters.put(painlessField.name, painlessField.getter);
-            painlessClassBuilder.setters.put(painlessField.name, painlessField.setter);
+        for (PainlessField painlessField : painlessClassBuilder.fields.values()) {
+            painlessClassBuilder.getterMethodHandles.put(painlessField.name, painlessField.getter);
+            painlessClassBuilder.setterMethodHandles.put(painlessField.name, painlessField.setter);
         }
     }
 
     private void setFunctionalInterfaceMethods() {
         for (Map.Entry<Class<?>, PainlessClassBuilder> painlessClassBuilderEntry : classesToPainlessClassBuilders.entrySet()) {
-            setFunctionalInterfaceMethod(painlessClassBuilderEntry.getValue());
+            setFunctionalInterfaceMethod(painlessClassBuilderEntry.getKey(), painlessClassBuilderEntry.getValue());
         }
     }
 
-    private void setFunctionalInterfaceMethod(PainlessClassBuilder painlessClassBuilder) {
-        Class<?> targetClass = painlessClassBuilder.clazz;
-
+    private void setFunctionalInterfaceMethod(Class<?> targetClass, PainlessClassBuilder painlessClassBuilder) {
         if (targetClass.isInterface()) {
             List<java.lang.reflect.Method> javaMethods = new ArrayList<>();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index c0d4433f7fb..f092a17c9fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -72,8 +72,9 @@ public final class ENewObj extends AExpression {
             constructor.arguments.toArray(types);
 
             if (constructor.arguments.size() != arguments.size()) {
-                throw createError(new IllegalArgumentException("When calling constructor on type [" + struct.name + "]" +
-                    " expected [" + constructor.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
+                throw createError(new IllegalArgumentException(
+                        "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] " +
+                        "expected [" + constructor.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
             }
 
             for (int argument = 0; argument < arguments.size(); ++argument) {
@@ -87,7 +88,8 @@ public final class ENewObj extends AExpression {
 
             statement = true;
         } else {
-            throw createError(new IllegalArgumentException("Unknown new call on type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException(
+                    "Unknown new call on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "]."));
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
index 7b55cb5a804..26471f67f65 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java
@@ -63,9 +63,9 @@ public final class PBrace extends AStoreable {
         } else if (prefix.actual == def.class) {
             sub = new PSubDefArray(location, index);
         } else if (Map.class.isAssignableFrom(prefix.actual)) {
-            sub = new PSubMapShortcut(location, locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual), index);
+            sub = new PSubMapShortcut(location, prefix.actual, index);
         } else if (List.class.isAssignableFrom(prefix.actual)) {
-            sub = new PSubListShortcut(location, locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual), index);
+            sub = new PSubListShortcut(location, prefix.actual, index);
         } else {
             throw createError(new IllegalArgumentException("Illegal array access on type " +
                     "[" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
index 8fc8a612b84..56bc18eadbd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java
@@ -84,8 +84,8 @@ public final class PCallInvoke extends AExpression {
         } else if (prefix.actual == def.class) {
             sub = new PSubDefCall(location, name, arguments);
         } else {
-            throw createError(new IllegalArgumentException(
-                "Unknown call [" + name + "] with [" + arguments.size() + "] arguments on type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments " +
+                    "on type [" + PainlessLookupUtility.typeToCanonicalTypeName(prefix.actual) + "]."));
         }
 
         if (nullSafe) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
index abf398d0e67..b322d5b1f28 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java
@@ -68,7 +68,7 @@ public final class PField extends AStoreable {
             sub = new PSubDefField(location, value);
         } else {
             PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual);
-            PainlessField field = prefix instanceof EStatic ? struct.staticMembers.get(value) : struct.members.get(value);
+            PainlessField field = prefix instanceof EStatic ? struct.staticFields.get(value) : struct.fields.get(value);
 
             if (field != null) {
                 sub = new PSubField(location, field);
@@ -92,11 +92,11 @@ public final class PField extends AStoreable {
                     index.analyze(locals);
 
                     if (Map.class.isAssignableFrom(prefix.actual)) {
-                        sub = new PSubMapShortcut(location, struct, index);
+                        sub = new PSubMapShortcut(location, prefix.actual, index);
                     }
 
                     if (List.class.isAssignableFrom(prefix.actual)) {
-                        sub = new PSubListShortcut(location, struct, index);
+                        sub = new PSubListShortcut(location, prefix.actual, index);
                     }
                 }
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index 3841b1fece1..509aad64153 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -36,16 +36,16 @@ import java.util.Set;
  */
 final class PSubListShortcut extends AStoreable {
 
-    private final PainlessClass struct;
+    private final Class<?> targetClass;
     private AExpression index;
 
     private PainlessMethod getter;
     private PainlessMethod setter;
 
-    PSubListShortcut(Location location, PainlessClass struct, AExpression index) {
+    PSubListShortcut(Location location, Class<?> targetClass, AExpression index) {
         super(location);
 
-        this.struct = Objects.requireNonNull(struct);
+        this.targetClass = Objects.requireNonNull(targetClass);
         this.index = Objects.requireNonNull(index);
     }
 
@@ -56,16 +56,19 @@ final class PSubListShortcut extends AStoreable {
 
     @Override
     void analyze(Locals locals) {
+        PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(targetClass);
+        String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
+
         getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1));
         setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("set", 2));
 
         if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1 ||
             getter.arguments.get(0) != int.class)) {
-            throw createError(new IllegalArgumentException("Illegal list get shortcut for type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal list get shortcut for type [" + canonicalClassName + "]."));
         }
 
         if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0) != int.class)) {
-            throw createError(new IllegalArgumentException("Illegal list set shortcut for type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal list set shortcut for type [" + canonicalClassName + "]."));
         }
 
         if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
@@ -80,7 +83,7 @@ final class PSubListShortcut extends AStoreable {
 
             actual = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
-            throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + canonicalClassName + "]."));
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index 13a3b9c9b94..2d7f2250c6c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -35,16 +35,16 @@ import java.util.Set;
  */
 final class PSubMapShortcut extends AStoreable {
 
-    private final PainlessClass struct;
+    private final Class<?> targetClass;
     private AExpression index;
 
     private PainlessMethod getter;
     private PainlessMethod setter;
 
-    PSubMapShortcut(Location location, PainlessClass struct, AExpression index) {
+    PSubMapShortcut(Location location, Class<?> targetClass, AExpression index) {
         super(location);
 
-        this.struct = Objects.requireNonNull(struct);
+        this.targetClass = Objects.requireNonNull(targetClass);
         this.index = Objects.requireNonNull(index);
     }
 
@@ -55,15 +55,18 @@ final class PSubMapShortcut extends AStoreable {
 
     @Override
     void analyze(Locals locals) {
+        PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(targetClass);
+        String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass);
+
         getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1));
         setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("put", 2));
 
         if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1)) {
-            throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + canonicalClassName + "]."));
         }
 
         if (setter != null && setter.arguments.size() != 2) {
-            throw createError(new IllegalArgumentException("Illegal map set shortcut for type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal map set shortcut for type [" + canonicalClassName + "]."));
         }
 
         if (getter != null && setter != null &&
@@ -78,7 +81,7 @@ final class PSubMapShortcut extends AStoreable {
 
             actual = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
-            throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + canonicalClassName + "]."));
         }
     }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index b2cc5e48ad8..ff0d4231175 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -29,6 +29,7 @@ import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.elasticsearch.painless.lookup.def;
 import org.elasticsearch.painless.spi.Whitelist;
 
 import java.io.IOException;
@@ -71,52 +72,54 @@ public class PainlessDocGenerator {
             Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
             false, StandardCharsets.UTF_8.name())) {
             emitGeneratedWarning(indexStream);
-            List<PainlessClass> structs = PAINLESS_LOOKUP.getStructs().stream().sorted(comparing(t -> t.name)).collect(toList());
-            for (PainlessClass struct : structs) {
-                if (struct.clazz.isPrimitive()) {
+            List<Class<?>> classes = PAINLESS_LOOKUP.getStructs().stream().sorted(comparing(Class::getCanonicalName)).collect(toList());
+            for (Class<?> clazz : classes) {
+                PainlessClass struct = PAINLESS_LOOKUP.getPainlessStructFromJavaClass(clazz);
+                String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(clazz);
+
+                if (clazz.isPrimitive()) {
                     // Primitives don't have methods to reference
                     continue;
                 }
-                if ("def".equals(struct.name)) {
+                if (clazz == def.class) {
                     // def is special but doesn't have any methods all of its own.
                     continue;
                 }
                 indexStream.print("include::");
-                indexStream.print(struct.name);
+                indexStream.print(canonicalClassName);
                 indexStream.println(".asciidoc[]");
 
-                Path typePath = apiRootPath.resolve(struct.name + ".asciidoc");
-                logger.info("Writing [{}.asciidoc]", struct.name);
+                Path typePath = apiRootPath.resolve(canonicalClassName + ".asciidoc");
+                logger.info("Writing [{}.asciidoc]", canonicalClassName);
                 try (PrintStream typeStream = new PrintStream(
                         Files.newOutputStream(typePath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
                         false, StandardCharsets.UTF_8.name())) {
                     emitGeneratedWarning(typeStream);
                     typeStream.print("[[");
-                    emitAnchor(typeStream, struct.clazz);
+                    emitAnchor(typeStream, clazz);
                     typeStream.print("]]++");
-                    typeStream.print(struct.name);
+                    typeStream.print(canonicalClassName);
                     typeStream.println("++::");
 
                     Consumer<PainlessField> documentField = field -> PainlessDocGenerator.documentField(typeStream, field);
                     Consumer<PainlessMethod> documentMethod = method -> PainlessDocGenerator.documentMethod(typeStream, method);
-                    struct.staticMembers.values().stream().sorted(FIELD_NAME).forEach(documentField);
-                    struct.members.values().stream().sorted(FIELD_NAME).forEach(documentField);
+                    struct.staticFields.values().stream().sorted(FIELD_NAME).forEach(documentField);
+                    struct.fields.values().stream().sorted(FIELD_NAME).forEach(documentField);
                     struct.staticMethods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(documentMethod);
                     struct.constructors.values().stream().sorted(NUMBER_OF_ARGS).forEach(documentMethod);
-                    Map<String, PainlessClass> inherited = new TreeMap<>();
+                    Map<String, Class<?>> inherited = new TreeMap<>();
                     struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(method -> {
-                        if (method.target == struct.clazz) {
+                        if (method.target == clazz) {
                             documentMethod(typeStream, method);
                         } else {
-                            PainlessClass painlessClass = PAINLESS_LOOKUP.getPainlessStructFromJavaClass(method.target);
-                            inherited.put(painlessClass.name, painlessClass);
+                            inherited.put(canonicalClassName, method.target);
                         }
                     });
 
                     if (false == inherited.isEmpty()) {
                         typeStream.print("* Inherits methods from ");
                         boolean first = true;
-                        for (PainlessClass inheritsFrom : inherited.values()) {
+                        for (Class<?> inheritsFrom : inherited.values()) {
                             if (first) {
                                 first = false;
                             } else {
@@ -242,7 +245,7 @@ public class PainlessDocGenerator {
      an internal link with the text.
      */
     private static void emitType(PrintStream stream, Class<?> clazz) {
-        emitStruct(stream, PAINLESS_LOOKUP.getPainlessStructFromJavaClass(clazz));
+        emitStruct(stream, clazz);
         while ((clazz = clazz.getComponentType()) != null) {
             stream.print("[]");
         }
@@ -252,15 +255,17 @@ public class PainlessDocGenerator {
      * Emit a {@link PainlessClass}. If the {@linkplain PainlessClass} is primitive or def this just emits the name of the struct.
      * Otherwise this emits an internal link with the name.
      */
-    private static void emitStruct(PrintStream stream, PainlessClass struct) {
-        if (false == struct.clazz.isPrimitive() && false == struct.name.equals("def")) {
+    private static void emitStruct(PrintStream stream, Class<?> clazz) {
+        String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(clazz);
+
+        if (false == clazz.isPrimitive() && clazz != def.class) {
             stream.print("<<");
-            emitAnchor(stream, struct.clazz);
+            emitAnchor(stream, clazz);
             stream.print(',');
-            stream.print(struct.name);
+            stream.print(canonicalClassName);
             stream.print(">>");
         } else {
-            stream.print(struct.name);
+            stream.print(canonicalClassName);
         }
     }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index 84452b4843d..c64014d81a5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -460,7 +460,7 @@ public class NodeToStringTests extends ESTestCase {
     public void testPSubField() {
         Location l = new Location(getTestName(), 0);
         PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(Boolean.class);
-        PainlessField f = s.staticMembers.get("TRUE");
+        PainlessField f = s.staticFields.get("TRUE");
         PSubField node = new PSubField(l, f);
         node.prefix = new EStatic(l, "Boolean");
         assertEquals("(PSubField (EStatic Boolean) TRUE)", node.toString());
@@ -469,32 +469,28 @@ public class NodeToStringTests extends ESTestCase {
 
     public void testPSubListShortcut() {
         Location l = new Location(getTestName(), 0);
-        PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(List.class);
-        PSubListShortcut node = new PSubListShortcut(l, s, new EConstant(l, 1));
+        PSubListShortcut node = new PSubListShortcut(l, List.class, new EConstant(l, 1));
         node.prefix = new EVariable(l, "a");
         assertEquals("(PSubListShortcut (EVariable a) (EConstant Integer 1))", node.toString());
         assertEquals("(PSubNullSafeCallInvoke (PSubListShortcut (EVariable a) (EConstant Integer 1)))",
                 new PSubNullSafeCallInvoke(l, node).toString());
 
         l = new Location(getTestName(), 0);
-        s = painlessLookup.getPainlessStructFromJavaClass(List.class);
-        node = new PSubListShortcut(l, s, new EBinary(l, Operation.ADD, new EConstant(l, 1), new EConstant(l, 4)));
+        node = new PSubListShortcut(l, List.class, new EBinary(l, Operation.ADD, new EConstant(l, 1), new EConstant(l, 4)));
         node.prefix = new EVariable(l, "a");
         assertEquals("(PSubListShortcut (EVariable a) (EBinary (EConstant Integer 1) + (EConstant Integer 4)))", node.toString());
     }
 
     public void testPSubMapShortcut() {
         Location l = new Location(getTestName(), 0);
-        PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(Map.class);
-        PSubMapShortcut node = new PSubMapShortcut(l, s, new EConstant(l, "cat"));
+        PSubMapShortcut node = new PSubMapShortcut(l, Map.class, new EConstant(l, "cat"));
         node.prefix = new EVariable(l, "a");
         assertEquals("(PSubMapShortcut (EVariable a) (EConstant String 'cat'))", node.toString());
         assertEquals("(PSubNullSafeCallInvoke (PSubMapShortcut (EVariable a) (EConstant String 'cat')))",
                 new PSubNullSafeCallInvoke(l, node).toString());
 
         l = new Location(getTestName(), 1);
-        s = painlessLookup.getPainlessStructFromJavaClass(Map.class);
-        node = new PSubMapShortcut(l, s, new EBinary(l, Operation.ADD, new EConstant(l, 1), new EConstant(l, 4)));
+        node = new PSubMapShortcut(l, Map.class, new EBinary(l, Operation.ADD, new EConstant(l, 1), new EConstant(l, 4)));
         node.prefix = new EVariable(l, "a");
         assertEquals("(PSubMapShortcut (EVariable a) (EBinary (EConstant Integer 1) + (EConstant Integer 4)))", node.toString());
     }

From 7a56df7c98918d78454a7da8bfe1795227b6c8e2 Mon Sep 17 00:00:00 2001
From: Tim Brooks <tim@uncontended.net>
Date: Thu, 26 Jul 2018 10:06:24 -0600
Subject: [PATCH 196/260] Release requests in cors handler (#32364)

There are two scenarios where a http request could terminate in the cors
handler. If that occurs, the requests need to be released. This commit
releases those requests.
---
 .../http/netty4/cors/Netty4CorsHandler.java   | 29 ++++++++++++++-----
 .../http/nio/cors/NioCorsHandler.java         | 29 ++++++++++++++-----
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java
index 3dad3c8a437..78ea9decd1d 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java
@@ -24,6 +24,7 @@ import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelPromise;
 import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.HttpHeaderNames;
 import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpMethod;
@@ -50,7 +51,7 @@ public class Netty4CorsHandler extends ChannelDuplexHandler {
     private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://");
 
     private final Netty4CorsConfig config;
-    private HttpRequest request;
+    private FullHttpRequest request;
 
     /**
      * Creates a new instance with the specified {@link Netty4CorsConfig}.
@@ -64,15 +65,24 @@ public class Netty4CorsHandler extends ChannelDuplexHandler {
 
     @Override
     public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
-        if (config.isCorsSupportEnabled() && msg instanceof HttpRequest) {
-            request = (HttpRequest) msg;
+        assert msg instanceof FullHttpRequest : "Invalid message type: " + msg.getClass();
+        if (config.isCorsSupportEnabled()) {
+            request = (FullHttpRequest) msg;
             if (isPreflightRequest(request)) {
-                handlePreflight(ctx, request);
-                return;
+                try {
+                    handlePreflight(ctx, request);
+                    return;
+                } finally {
+                    releaseRequest();
+                }
             }
             if (config.isShortCircuit() && !validateOrigin()) {
-                forbidden(ctx, request);
-                return;
+                try {
+                    forbidden(ctx, request);
+                    return;
+                } finally {
+                    releaseRequest();
+                }
             }
         }
         ctx.fireChannelRead(msg);
@@ -123,6 +133,11 @@ public class Netty4CorsHandler extends ChannelDuplexHandler {
         }
     }
 
+    private void releaseRequest() {
+        request.release();
+        request = null;
+    }
+
     private static void forbidden(final ChannelHandlerContext ctx, final HttpRequest request) {
         ctx.writeAndFlush(new DefaultFullHttpResponse(request.protocolVersion(), HttpResponseStatus.FORBIDDEN))
             .addListener(ChannelFutureListener.CLOSE);
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java
index 98ae2d523ca..5a9d114d675 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java
@@ -24,6 +24,7 @@ import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelPromise;
 import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.HttpHeaderNames;
 import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpMethod;
@@ -50,7 +51,7 @@ public class NioCorsHandler extends ChannelDuplexHandler {
     private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://");
 
     private final NioCorsConfig config;
-    private HttpRequest request;
+    private FullHttpRequest request;
 
     /**
      * Creates a new instance with the specified {@link NioCorsConfig}.
@@ -64,15 +65,24 @@ public class NioCorsHandler extends ChannelDuplexHandler {
 
     @Override
     public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
-        if (config.isCorsSupportEnabled() && msg instanceof HttpRequest) {
-            request = (HttpRequest) msg;
+        assert msg instanceof FullHttpRequest : "Invalid message type: " + msg.getClass();
+        if (config.isCorsSupportEnabled()) {
+            request = (FullHttpRequest) msg;
             if (isPreflightRequest(request)) {
-                handlePreflight(ctx, request);
-                return;
+                try {
+                    handlePreflight(ctx, request);
+                    return;
+                } finally {
+                    releaseRequest();
+                }
             }
             if (config.isShortCircuit() && !validateOrigin()) {
-                forbidden(ctx, request);
-                return;
+                try {
+                    forbidden(ctx, request);
+                    return;
+                } finally {
+                    releaseRequest();
+                }
             }
         }
         ctx.fireChannelRead(msg);
@@ -109,6 +119,11 @@ public class NioCorsHandler extends ChannelDuplexHandler {
         }
     }
 
+    private void releaseRequest() {
+        request.release();
+        request = null;
+    }
+
     private void handlePreflight(final ChannelHandlerContext ctx, final HttpRequest request) {
         final HttpResponse response = new DefaultFullHttpResponse(request.protocolVersion(), HttpResponseStatus.OK, true, true);
         if (setOrigin(response)) {

From 11a2189c6f738fe0d2773eba49c6feca8e377dfa Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 26 Jul 2018 12:11:31 -0400
Subject: [PATCH 197/260] Switch x-pack full restart to new style Requests
 (#32294)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `x-pack:qa:full-cluster-restart` project to use
the new versions.
---
 .../xpack/restart/FullClusterRestartIT.java   | 161 +++++++++---------
 1 file changed, 78 insertions(+), 83 deletions(-)

diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index ba6f9e91678..b91092fe9e7 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -5,8 +5,6 @@
  */
 package org.elasticsearch.xpack.restart;
 
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
@@ -15,9 +13,7 @@ import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.StreamsUtils;
@@ -37,16 +33,12 @@ import org.junit.Before;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
@@ -110,11 +102,13 @@ public class FullClusterRestartIT extends ESRestTestCase {
         String doc = "{\"test\": \"test\"}";
 
         if (runningAgainstOldCluster) {
-            client().performRequest("PUT", docLocation, singletonMap("refresh", "true"),
-                    new StringEntity(doc, ContentType.APPLICATION_JSON));
+            Request createDoc = new Request("PUT", docLocation);
+            createDoc.addParameter("refresh", "true");
+            createDoc.setJsonEntity(doc);
+            client().performRequest(createDoc);
         }
 
-        assertThat(toStr(client().performRequest("GET", docLocation)), containsString(doc));
+        assertThat(toStr(client().performRequest(new Request("GET", docLocation))), containsString(doc));
     }
 
     @SuppressWarnings("unchecked")
@@ -124,8 +118,8 @@ public class FullClusterRestartIT extends ESRestTestCase {
             createRole("preupgrade_role");
         } else {
             waitForYellow(".security");
-            Response settingsResponse = client().performRequest("GET", "/.security/_settings/index.format");
-            Map<String, Object> settingsResponseMap = toMap(settingsResponse);
+            Response settingsResponse = client().performRequest(new Request("GET", "/.security/_settings/index.format"));
+            Map<String, Object> settingsResponseMap = entityAsMap(settingsResponse);
             logger.info("settings response map {}", settingsResponseMap);
             final boolean needsUpgrade;
             final String concreteSecurityIndex;
@@ -157,7 +151,8 @@ public class FullClusterRestartIT extends ESRestTestCase {
                             "the native realm will not be operational until the upgrade API is run on the security index"));
                 }
                 // run upgrade API
-                Response upgradeResponse = client().performRequest("POST", "_xpack/migration/upgrade/" + concreteSecurityIndex);
+                Response upgradeResponse = client().performRequest(
+                        new Request("POST", "_xpack/migration/upgrade/" + concreteSecurityIndex));
                 logger.info("upgrade response:\n{}", toStr(upgradeResponse));
             }
 
@@ -177,16 +172,19 @@ public class FullClusterRestartIT extends ESRestTestCase {
     public void testWatcher() throws Exception {
         if (runningAgainstOldCluster) {
             logger.info("Adding a watch on old cluster {}", oldClusterVersion);
-            client().performRequest("PUT", "_xpack/watcher/watch/bwc_watch", emptyMap(),
-                    new StringEntity(loadWatch("simple-watch.json"), ContentType.APPLICATION_JSON));
+            Request createBwcWatch = new Request("PUT", "_xpack/watcher/watch/bwc_watch");
+            createBwcWatch.setJsonEntity(loadWatch("simple-watch.json"));
+            client().performRequest(createBwcWatch);
 
             logger.info("Adding a watch with \"fun\" throttle periods on old cluster");
-            client().performRequest("PUT", "_xpack/watcher/watch/bwc_throttle_period", emptyMap(),
-                    new StringEntity(loadWatch("throttle-period-watch.json"), ContentType.APPLICATION_JSON));
+            Request createBwcThrottlePeriod = new Request("PUT", "_xpack/watcher/watch/bwc_throttle_period");
+            createBwcThrottlePeriod.setJsonEntity(loadWatch("throttle-period-watch.json"));
+            client().performRequest(createBwcThrottlePeriod);
 
             logger.info("Adding a watch with \"fun\" read timeout on old cluster");
-            client().performRequest("PUT", "_xpack/watcher/watch/bwc_funny_timeout", emptyMap(),
-                    new StringEntity(loadWatch("funny-timeout-watch.json"), ContentType.APPLICATION_JSON));
+            Request createFunnyTimeout = new Request("PUT", "_xpack/watcher/watch/bwc_funny_timeout");
+            createFunnyTimeout.setJsonEntity(loadWatch("funny-timeout-watch.json"));
+            client().performRequest(createFunnyTimeout);
 
             logger.info("Waiting for watch results index to fill up...");
             waitForYellow(".watches,bwc_watch_index,.watcher-history*");
@@ -198,7 +196,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
             waitForYellow(".watches,bwc_watch_index,.watcher-history*");
 
             logger.info("checking if the upgrade procedure on the new cluster is required");
-            Map<String, Object> response = toMap(client().performRequest("GET", "/_xpack/migration/assistance"));
+            Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/_xpack/migration/assistance")));
             logger.info(response);
 
             @SuppressWarnings("unchecked") Map<String, Object> indices = (Map<String, Object>) response.get("indices");
@@ -211,14 +209,16 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
                 logger.info("starting upgrade procedure on the new cluster");
 
-                Map<String, String> params = Collections.singletonMap("error_trace", "true");
-                Map<String, Object> upgradeResponse = toMap(client().performRequest("POST", "_xpack/migration/upgrade/.watches", params));
+                Request migrationAssistantRequest = new Request("POST", "_xpack/migration/upgrade/.watches");
+                migrationAssistantRequest.addParameter("error_trace", "true");
+                Map<String, Object> upgradeResponse = entityAsMap(client().performRequest(migrationAssistantRequest));
                 assertThat(upgradeResponse.get("timed_out"), equalTo(Boolean.FALSE));
                 // we posted 3 watches, but monitoring can post a few more
                 assertThat((int) upgradeResponse.get("total"), greaterThanOrEqualTo(3));
 
                 logger.info("checking that upgrade procedure on the new cluster is no longer required");
-                Map<String, Object> responseAfter = toMap(client().performRequest("GET", "/_xpack/migration/assistance"));
+                Map<String, Object> responseAfter = entityAsMap(client().performRequest(
+                        new Request("GET", "/_xpack/migration/assistance")));
                 @SuppressWarnings("unchecked") Map<String, Object> indicesAfter = (Map<String, Object>) responseAfter.get("indices");
                 assertNull(indicesAfter.get(".watches"));
             } else {
@@ -226,10 +226,10 @@ public class FullClusterRestartIT extends ESRestTestCase {
             }
 
             // Wait for watcher to actually start....
-            Map<String, Object> startWatchResponse = toMap(client().performRequest("POST", "_xpack/watcher/_start"));
+            Map<String, Object> startWatchResponse = entityAsMap(client().performRequest(new Request("POST", "_xpack/watcher/_start")));
             assertThat(startWatchResponse.get("acknowledged"), equalTo(Boolean.TRUE));
             assertBusy(() -> {
-                Map<String, Object> statsWatchResponse = toMap(client().performRequest("GET", "_xpack/watcher/stats"));
+                Map<String, Object> statsWatchResponse = entityAsMap(client().performRequest(new Request("GET", "_xpack/watcher/stats")));
                 @SuppressWarnings("unchecked")
                 List<Object> states = ((List<Object>) statsWatchResponse.get("stats"))
                         .stream().map(o -> ((Map<String, Object>) o).get("watcher_state")).collect(Collectors.toList());
@@ -244,10 +244,11 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 /* Shut down watcher after every test because watcher can be a bit finicky about shutting down when the node shuts
                  * down. This makes super sure it shuts down *and* causes the test to fail in a sensible spot if it doesn't shut down.
                  */
-                Map<String, Object> stopWatchResponse = toMap(client().performRequest("POST", "_xpack/watcher/_stop"));
+                Map<String, Object> stopWatchResponse = entityAsMap(client().performRequest(new Request("POST", "_xpack/watcher/_stop")));
                 assertThat(stopWatchResponse.get("acknowledged"), equalTo(Boolean.TRUE));
                 assertBusy(() -> {
-                    Map<String, Object> statsStoppedWatchResponse = toMap(client().performRequest("GET", "_xpack/watcher/stats"));
+                    Map<String, Object> statsStoppedWatchResponse = entityAsMap(client().performRequest(
+                            new Request("GET", "_xpack/watcher/stats")));
                     @SuppressWarnings("unchecked")
                     List<Object> states = ((List<Object>) statsStoppedWatchResponse.get("stats"))
                             .stream().map(o -> ((Map<String, Object>) o).get("watcher_state")).collect(Collectors.toList());
@@ -297,12 +298,12 @@ public class FullClusterRestartIT extends ESRestTestCase {
                     + "]"
                     + "}");
 
-            Map<String, Object> createRollupJobResponse = toMap(client().performRequest(createRollupJobRequest));
+            Map<String, Object> createRollupJobResponse = entityAsMap(client().performRequest(createRollupJobRequest));
             assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE));
 
             // start the rollup job
             final Request startRollupJobRequest = new Request("POST", "_xpack/rollup/job/rollup-job-test/_start");
-            Map<String, Object> startRollupJobResponse = toMap(client().performRequest(startRollupJobRequest));
+            Map<String, Object> startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest));
             assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
 
             assertRollUpJob("rollup-job-test");
@@ -315,7 +316,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
             if (oldClusterVersion.onOrAfter(Version.V_6_2_0)) {
                 clusterHealthRequest.addParameter("wait_for_no_initializing_shards", "true");
             }
-            Map<String, Object> clusterHealthResponse = toMap(client().performRequest(clusterHealthRequest));
+            Map<String, Object> clusterHealthResponse = entityAsMap(client().performRequest(clusterHealthRequest));
             assertThat(clusterHealthResponse.get("timed_out"), equalTo(Boolean.FALSE));
 
             assertRollUpJob("rollup-job-test");
@@ -327,14 +328,17 @@ public class FullClusterRestartIT extends ESRestTestCase {
         assumeTrue("It is only possible to build an index that sql doesn't like before 6.0.0",
                 oldClusterVersion.before(Version.V_6_0_0_alpha1));
         if (runningAgainstOldCluster) {
-            client().performRequest("POST", "/testsqlfailsonindexwithtwotypes/type1", emptyMap(),
-                    new StringEntity("{}", ContentType.APPLICATION_JSON));
-            client().performRequest("POST", "/testsqlfailsonindexwithtwotypes/type2", emptyMap(),
-                    new StringEntity("{}", ContentType.APPLICATION_JSON));
+            Request doc1 = new Request("POST", "/testsqlfailsonindexwithtwotypes/type1");
+            doc1.setJsonEntity("{}");
+            client().performRequest(doc1);
+            Request doc2 = new Request("POST", "/testsqlfailsonindexwithtwotypes/type2");
+            doc2.setJsonEntity("{}");
+            client().performRequest(doc2);
             return;
         }
-        ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest("POST", "/_xpack/sql", emptyMap(),
-                new StringEntity("{\"query\":\"SELECT * FROM testsqlfailsonindexwithtwotypes\"}", ContentType.APPLICATION_JSON)));
+        Request sqlRequest = new Request("POST", "/_xpack/sql");
+        sqlRequest.setJsonEntity("{\"query\":\"SELECT * FROM testsqlfailsonindexwithtwotypes\"}");
+        ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(sqlRequest));
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString(
             "[testsqlfailsonindexwithtwotypes] contains more than one type [type1, type2] so it is incompatible with sql"));
@@ -346,14 +350,14 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
     @SuppressWarnings("unchecked")
     private void assertOldTemplatesAreDeleted() throws IOException {
-        Map<String, Object> templates = toMap(client().performRequest("GET", "/_template"));
+        Map<String, Object> templates = entityAsMap(client().performRequest(new Request("GET", "/_template")));
         assertThat(templates.keySet(), not(hasItems(is("watches"), startsWith("watch-history"), is("triggered_watches"))));
     }
 
     @SuppressWarnings("unchecked")
     private void assertWatchIndexContentsWork() throws Exception {
         // Fetch a basic watch
-        Map<String, Object> bwcWatch = toMap(client().performRequest("GET", "_xpack/watcher/watch/bwc_watch"));
+        Map<String, Object> bwcWatch = entityAsMap(client().performRequest(new Request("GET", "_xpack/watcher/watch/bwc_watch")));
 
         logger.error("-----> {}", bwcWatch);
 
@@ -368,7 +372,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
         assertThat(ObjectPath.eval("actions.index_payload.index.timeout_in_millis", source), equalTo(timeout));
 
         // Fetch a watch with "fun" throttle periods
-        bwcWatch = toMap(client().performRequest("GET", "_xpack/watcher/watch/bwc_throttle_period"));
+        bwcWatch = entityAsMap(client().performRequest(new Request("GET", "_xpack/watcher/watch/bwc_throttle_period")));
         assertThat(bwcWatch.get("found"), equalTo(true));
         source = (Map<String, Object>) bwcWatch.get("watch");
         assertEquals(timeout, source.get("throttle_period_in_millis"));
@@ -378,7 +382,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
          * Fetch a watch with a funny timeout to verify loading fractional time
          * values.
          */
-        bwcWatch = toMap(client().performRequest("GET", "_xpack/watcher/watch/bwc_funny_timeout"));
+        bwcWatch = entityAsMap(client().performRequest(new Request("GET", "_xpack/watcher/watch/bwc_funny_timeout")));
         assertThat(bwcWatch.get("found"), equalTo(true));
         source = (Map<String, Object>) bwcWatch.get("watch");
 
@@ -396,7 +400,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
         // password doesn't come back because it is hidden
         assertThat(basic, hasEntry(is("password"), anyOf(startsWith("::es_encrypted::"), is("::es_redacted::"))));
 
-        Map<String, Object> history = toMap(client().performRequest("GET", ".watcher-history*/_search"));
+        Map<String, Object> history = entityAsMap(client().performRequest(new Request("GET", ".watcher-history*/_search")));
         Map<String, Object> hits = (Map<String, Object>) history.get("hits");
         assertThat((int) (hits.get("total")), greaterThanOrEqualTo(2));
     }
@@ -407,20 +411,20 @@ public class FullClusterRestartIT extends ESRestTestCase {
                 .condition(InternalAlwaysCondition.INSTANCE)
                 .trigger(ScheduleTrigger.builder(new IntervalSchedule(IntervalSchedule.Interval.seconds(1))))
                 .addAction("awesome", LoggingAction.builder(new TextTemplate("test"))).buildAsBytes(XContentType.JSON).utf8ToString();
-        Map<String, Object> put = toMap(client().performRequest("PUT", "_xpack/watcher/watch/new_watch", emptyMap(),
-                new StringEntity(watch, ContentType.APPLICATION_JSON)));
+        Request createWatchRequest = new Request("PUT", "_xpack/watcher/watch/new_watch");
+        createWatchRequest.setJsonEntity(watch);
+        Map<String, Object> createWatch = entityAsMap(client().performRequest(createWatchRequest));
 
-        logger.info(put);
+        logger.info("create watch {}", createWatch);
 
-        assertThat(put.get("created"), equalTo(true));
-        assertThat(put.get("_version"), equalTo(1));
+        assertThat(createWatch.get("created"), equalTo(true));
+        assertThat(createWatch.get("_version"), equalTo(1));
 
-        put = toMap(client().performRequest("PUT", "_xpack/watcher/watch/new_watch", emptyMap(),
-                new StringEntity(watch, ContentType.APPLICATION_JSON)));
-        assertThat(put.get("created"), equalTo(false));
-        assertThat(put.get("_version"), equalTo(2));
+        Map<String, Object> updateWatch = entityAsMap(client().performRequest(createWatchRequest));
+        assertThat(updateWatch.get("created"), equalTo(false));
+        assertThat(updateWatch.get("_version"), equalTo(2));
 
-        Map<String, Object> get = toMap(client().performRequest("GET", "_xpack/watcher/watch/new_watch"));
+        Map<String, Object> get = entityAsMap(client().performRequest(new Request("GET", "_xpack/watcher/watch/new_watch")));
         assertThat(get.get("found"), equalTo(true));
         @SuppressWarnings("unchecked") Map<?, ?> source = (Map<String, Object>) get.get("watch");
         Map<String, Object>  logging = ObjectPath.eval("actions.awesome.logging", source);
@@ -429,23 +433,24 @@ public class FullClusterRestartIT extends ESRestTestCase {
     }
 
     private void waitForYellow(String indexName) throws IOException {
-        Map<String, String> params = new HashMap<>();
-        params.put("wait_for_status", "yellow");
-        params.put("timeout", "30s");
-        params.put("wait_for_no_relocating_shards", "true");
+        Request request = new Request("GET", "/_cluster/health/" + indexName);
+        request.addParameter("wait_for_status", "yellow");
+        request.addParameter("timeout", "30s");
+        request.addParameter("wait_for_no_relocating_shards", "true");
         if (oldClusterVersion.onOrAfter(Version.V_6_2_0)) {
-            params.put("wait_for_no_initializing_shards", "true");
+            request.addParameter("wait_for_no_initializing_shards", "true");
         }
-        Map<String, Object> response = toMap(client().performRequest("GET", "/_cluster/health/" + indexName, params));
+        Map<String, Object> response = entityAsMap(client().performRequest(request));
         assertThat(response.get("timed_out"), equalTo(Boolean.FALSE));
     }
 
     @SuppressWarnings("unchecked")
     private void waitForHits(String indexName, int expectedHits) throws Exception {
-        Map<String, String> params = singletonMap("size", "0");
+        Request request = new Request("GET", "/" + indexName + "/_search");
+        request.addParameter("size", "0");
         assertBusy(() -> {
             try {
-                Map<String, Object> response = toMap(client().performRequest("GET", "/" + indexName + "/_search", params));
+                Map<String, Object> response = entityAsMap(client().performRequest(request));
                 Map<String, Object> hits = (Map<String, Object>) response.get("hits");
                 int total = (int) hits.get("total");
                 assertThat(total, greaterThanOrEqualTo(expectedHits));
@@ -461,34 +466,26 @@ public class FullClusterRestartIT extends ESRestTestCase {
         }, 30, TimeUnit.SECONDS);
     }
 
-    static Map<String, Object> toMap(Response response) throws IOException {
-        return toMap(EntityUtils.toString(response.getEntity()));
-    }
-
-    static Map<String, Object> toMap(String response) throws IOException {
-        return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false);
-    }
-
     static String toStr(Response response) throws IOException {
         return EntityUtils.toString(response.getEntity());
     }
 
     private void createUser(final String id) throws Exception {
-        final String userJson =
+        Request request = new Request("PUT", "/_xpack/security/user/" + id);
+        request.setJsonEntity(
             "{\n" +
             "   \"password\" : \"j@rV1s\",\n" +
             "   \"roles\" : [ \"admin\", \"other_role1\" ],\n" +
             "   \"full_name\" : \"" + randomAlphaOfLength(5) + "\",\n" +
             "   \"email\" : \"" + id + "@example.com\",\n" +
             "   \"enabled\": true\n" +
-            "}";
-
-        client().performRequest("PUT", "/_xpack/security/user/" + id, emptyMap(),
-            new StringEntity(userJson, ContentType.APPLICATION_JSON));
+            "}");
+        client().performRequest(request);
     }
 
     private void createRole(final String id) throws Exception {
-        final String roleJson =
+        Request request = new Request("PUT", "/_xpack/security/role/" + id);
+        request.setJsonEntity(
             "{\n" +
             "  \"run_as\": [ \"abc\" ],\n" +
             "  \"cluster\": [ \"monitor\" ],\n" +
@@ -502,14 +499,12 @@ public class FullClusterRestartIT extends ESRestTestCase {
             "      \"query\": \"{\\\"match\\\": {\\\"category\\\": \\\"click\\\"}}\"\n" +
             "    }\n" +
             "  ]\n" +
-            "}";
-
-        client().performRequest("PUT", "/_xpack/security/role/" + id, emptyMap(),
-            new StringEntity(roleJson, ContentType.APPLICATION_JSON));
+            "}");
+        client().performRequest(request);
     }
 
     private void assertUserInfo(final String user) throws Exception {
-        Map<String, Object> response = toMap(client().performRequest("GET", "/_xpack/security/user/" + user));
+        Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/_xpack/security/user/" + user)));
         @SuppressWarnings("unchecked") Map<String, Object> userInfo = (Map<String, Object>) response.get(user);
         assertEquals(user + "@example.com", userInfo.get("email"));
         assertNotNull(userInfo.get("full_name"));
@@ -518,7 +513,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
     private void assertRoleInfo(final String role) throws Exception {
         @SuppressWarnings("unchecked") Map<String, Object> response = (Map<String, Object>)
-                toMap(client().performRequest("GET", "/_xpack/security/role/" + role)).get(role);
+                entityAsMap(client().performRequest(new Request("GET", "/_xpack/security/role/" + role))).get(role);
         assertNotNull(response.get("run_as"));
         assertNotNull(response.get("cluster"));
         assertNotNull(response.get("indices"));
@@ -531,7 +526,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
         // check that the rollup job is started using the RollUp API
         final Request getRollupJobRequest = new Request("GET", "_xpack/rollup/job/" + rollupJob);
-        Map<String, Object> getRollupJobResponse = toMap(client().performRequest(getRollupJobRequest));
+        Map<String, Object> getRollupJobResponse = entityAsMap(client().performRequest(getRollupJobRequest));
         Map<String, Object> job = getJob(getRollupJobResponse, rollupJob);
         if (job != null) {
             assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
@@ -541,7 +536,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
         final Request taskRequest = new Request("GET", "_tasks");
         taskRequest.addParameter("detailed", "true");
         taskRequest.addParameter("actions", "xpack/rollup/*");
-        Map<String, Object> taskResponse = toMap(client().performRequest(taskRequest));
+        Map<String, Object> taskResponse = entityAsMap(client().performRequest(taskRequest));
         Map<String, Object> taskResponseNodes = (Map<String, Object>) taskResponse.get("nodes");
         Map<String, Object> taskResponseNode = (Map<String, Object>) taskResponseNodes.values().iterator().next();
         Map<String, Object> taskResponseTasks = (Map<String, Object>) taskResponseNode.get("tasks");
@@ -550,7 +545,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
 
         // check that the rollup job is started using the Cluster State API
         final Request clusterStateRequest = new Request("GET", "_cluster/state/metadata");
-        Map<String, Object> clusterStateResponse = toMap(client().performRequest(clusterStateRequest));
+        Map<String, Object> clusterStateResponse = entityAsMap(client().performRequest(clusterStateRequest));
         List<Map<String, Object>> rollupJobTasks = ObjectPath.eval("metadata.persistent_tasks.tasks", clusterStateResponse);
 
         boolean hasRollupTask = false;

From eb73dde7c83f1e988104c2539763768d57c1454e Mon Sep 17 00:00:00 2001
From: Jack Conradson <jack@elastic.co>
Date: Thu, 26 Jul 2018 09:14:43 -0700
Subject: [PATCH 198/260] Fix missing JavaDoc for @throws in several places in
 KerberosTicketValidator.

---
 .../security/authc/kerberos/KerberosTicketValidator.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidator.java
index a63d90178dc..8482e6f090c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTicketValidator.java
@@ -132,7 +132,7 @@ public class KerberosTicketValidator {
      * @param subject authenticated subject
      * @return a byte[] containing the token to be sent to the peer. null indicates
      *         that no token is generated.
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      * @see GSSContext#acceptSecContext(byte[], int, int)
      */
     private static byte[] acceptSecContext(final byte[] base64decodedTicket, final GSSContext gssContext, Subject subject)
@@ -148,7 +148,7 @@ public class KerberosTicketValidator {
      * @param gssManager {@link GSSManager}
      * @param subject logged in {@link Subject}
      * @return {@link GSSCredential} for particular mechanism
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     private static GSSCredential createCredentials(final GSSManager gssManager, final Subject subject) throws PrivilegedActionException {
         return doAsWrapper(subject, (PrivilegedExceptionAction<GSSCredential>) () -> gssManager.createCredential(null,
@@ -163,7 +163,7 @@ public class KerberosTicketValidator {
      * @param action {@link PrivilegedExceptionAction} action for performing inside
      *            Subject.doAs
      * @return the value returned by the PrivilegedExceptionAction's run method
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     private static <T> T doAsWrapper(final Subject subject, final PrivilegedExceptionAction<T> action) throws PrivilegedActionException {
         try {

From 0ed3458534beb8d93852e57b2ee422324613c528 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 26 Jul 2018 12:35:15 -0400
Subject: [PATCH 199/260] TEST: Avoid triggering merges in FlushIT

In testSyncedFlushSkipOutOfSyncReplicas, we reindex the extra documents
to all shards including the out-of-sync replica. However, reindexing to
that replica can trigger merges (due to the new deletes) which cause the
synced-flush failed. This test starts failing after we aggressively
trigger merges segments with a large number of deletes in LUCENE-8263.
---
 .../test/java/org/elasticsearch/indices/flush/FlushIT.java | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
index a543e87adcb..185a9b5cdce 100644
--- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
@@ -274,8 +274,11 @@ public class FlushIT extends ESIntegTestCase {
             "out of sync replica; num docs on replica [" + (numDocs + extraDocs) + "]; num docs on primary [" + numDocs + "]"));
         // Index extra documents to all shards - synced-flush should be ok.
         for (IndexShard indexShard : indexShards) {
-            for (int i = 0; i < extraDocs; i++) {
-                indexDoc(IndexShardTestCase.getEngine(indexShard), "extra_" + i);
+            // Do reindex documents to the out of sync replica to avoid trigger merges
+            if (indexShard != outOfSyncReplica) {
+                for (int i = 0; i < extraDocs; i++) {
+                    indexDoc(IndexShardTestCase.getEngine(indexShard), "extra_" + i);
+                }
             }
         }
         final ShardsSyncedFlushResult fullResult = SyncedFlushUtil.attemptSyncedFlush(logger, internalCluster(), shardId);

From 0f783ce94b5f670e9877c37ba481d5c2f1e4d7ac Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 26 Jul 2018 13:19:05 -0400
Subject: [PATCH 200/260] Build: Stop double generating buildSrc pom (#32408)

When we added the `java-gradle-plugin` to `buildSrc` it added a second
task to generate the pom that duplicates the publishing work that we
configure in `BuildPlugin`. Not only does it dupliciate the pom, it
creates a pom that is missing things like `name` and `description` which
are required for publishing to maven central.

This change disables the duplicate pom generation.
---
 buildSrc/build.gradle | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index d507c54c45a..5775b2b6323 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -183,4 +183,12 @@ if (project != rootProject) {
     testClass = 'org.elasticsearch.gradle.test.GradleUnitTestCase'
     integTestClass = 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
   }
+
+  /*
+   * We alread configure publication and we don't need or want this one that
+   * comes from the java-gradle-plugin.
+   */
+  afterEvaluate {
+    generatePomFileForPluginMavenPublication.enabled = false
+  }
 }

From 7ad16ffd84046a200e0b7ef9a8ac0388e4d62f74 Mon Sep 17 00:00:00 2001
From: Sandeep Kanabar <sandeepkanabar@gmail.com>
Date: Thu, 26 Jul 2018 22:59:17 +0530
Subject: [PATCH 201/260] Docs: Correcting a typo in tophits (#32359)

---
 .../reference/aggregations/metrics/tophits-aggregation.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index dc3222a5f37..5eeb3a4605a 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -172,7 +172,7 @@ In the example below we search across crawled webpages. For each webpage we stor
 belong to. By defining a `terms` aggregator on the `domain` field we group the result set of webpages by domain. The
 `top_hits` aggregator is then defined as sub-aggregator, so that the top matching hits are collected per bucket.
 
-Also a `max` aggregator is defined which is used by the `terms` aggregator's order feature the return the buckets by
+Also a `max` aggregator is defined which is used by the `terms` aggregator's order feature to return the buckets by
 relevancy order of the most relevant document in a bucket.
 
 [source,js]

From 643235d46a80dd0201677fa685ed14c8a5a7c6de Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 26 Jul 2018 13:31:16 -0400
Subject: [PATCH 202/260] Switch x-pack/plugin to new style Requests (#32327)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `x-pack/plugin` project to use the new versions.
---
 .../elasticsearch/xpack/test/rest/XPackRestIT.java    | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
index f1495f4f3ac..31d6312f662 100644
--- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
+++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
@@ -9,6 +9,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.http.HttpStatus;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.settings.Settings;
@@ -128,18 +129,22 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase {
                     () -> "Exception when waiting for [" + template + "] template to be created");
             }
 
-            boolean existsWatcherIndex = adminClient().performRequest("HEAD", ".watches").getStatusLine().getStatusCode() == 200;
+            boolean existsWatcherIndex = adminClient()
+                    .performRequest(new Request("HEAD", ".watches"))
+                    .getStatusLine().getStatusCode() == 200;
             if (existsWatcherIndex == false) {
                 return;
             }
-            Response response = adminClient().performRequest("GET", ".watches/_search", Collections.singletonMap("size", "1000"));
+            Request searchWatchesRequest = new Request("GET", ".watches/_search");
+            searchWatchesRequest.addParameter("size", "1000");
+            Response response = adminClient().performRequest(searchWatchesRequest);
             ObjectPath objectPathResponse = ObjectPath.createFromResponse(response);
             int totalHits = objectPathResponse.evaluate("hits.total");
             if (totalHits > 0) {
                 List<Map<String, Object>> hits = objectPathResponse.evaluate("hits.hits");
                 for (Map<String, Object> hit : hits) {
                     String id = (String) hit.get("_id");
-                    assertOK(adminClient().performRequest("DELETE", "_xpack/watcher/watch/" + id));
+                    adminClient().performRequest(new Request("DELETE", "_xpack/watcher/watch/" + id));
                 }
             }
         }

From 57876bfeb916e18b57e0624a3dfed9779b109f14 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Thu, 26 Jul 2018 19:55:28 +0200
Subject: [PATCH 203/260] INGEST: Fix Deprecation Warning in Script Proc.
 (#32407)

* Using short script form normalized to a map that used 'inline' instead of 'source' so a short form processor definition like:

```
{
 "script": "ctx.foo= 'bar'"
}
```

would always warn about the following deprecation:

```
  #! Deprecation: Deprecated field [inline] used, expected [source]
```
---
 .../main/java/org/elasticsearch/ingest/ConfigurationUtils.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 78dc0ec6bfe..2853842c646 100644
--- a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -361,7 +361,7 @@ public final class ConfigurationUtils {
             return readProcessor(processorFactories, type, (Map<String, Object>) config);
         } else if (config instanceof String && "script".equals(type)) {
             Map<String, Object> normalizedScript = new HashMap<>(1);
-            normalizedScript.put(ScriptType.INLINE.getName(), config);
+            normalizedScript.put(ScriptType.INLINE.getParseField().getPreferredName(), config);
             return readProcessor(processorFactories, type, normalizedScript);
         } else {
             throw newConfigurationException(type, null, null,

From 8e5f281b27ccef436ad687b9ccb19b271b445ce9 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Thu, 26 Jul 2018 20:29:05 +0200
Subject: [PATCH 204/260] AbstractQueryTestCase should run without type less
 often (#28936)

This commit changes the randomization to always create an index with a type.
It also adds a way to create a query shard context that maps to an index with
no type registered in order to explicitely test cases where there is no type.
---
 .../index/query/FeatureQueryBuilderTests.java | 15 +--
 .../index/query/DisMaxQueryBuilderTests.java  |  1 -
 .../index/query/ExistsQueryBuilderTests.java  |  6 +-
 .../index/query/FuzzyQueryBuilderTests.java   |  5 -
 .../GeoBoundingBoxQueryBuilderTests.java      | 25 ++---
 .../query/GeoDistanceQueryBuilderTests.java   | 20 ----
 .../query/GeoPolygonQueryBuilderTests.java    | 22 -----
 .../query/GeoShapeQueryBuilderTests.java      | 12 ---
 .../index/query/IdsQueryBuilderTests.java     | 28 +++---
 .../MatchPhrasePrefixQueryBuilderTests.java   |  9 +-
 .../query/MatchPhraseQueryBuilderTests.java   |  8 +-
 .../index/query/MatchQueryBuilderTests.java   | 12 +--
 .../query/MoreLikeThisQueryBuilderTests.java  |  1 -
 .../query/MultiMatchQueryBuilderTests.java    | 14 ---
 .../index/query/PrefixQueryBuilderTests.java  |  1 -
 .../query/QueryStringQueryBuilderTests.java   | 50 ++--------
 .../index/query/RangeQueryBuilderTests.java   | 52 ++++------
 .../index/query/RegexpQueryBuilderTests.java  |  1 -
 .../query/SimpleQueryStringBuilderTests.java  | 25 +----
 .../index/query/TermQueryBuilderTests.java    |  1 -
 .../index/query/TermsQueryBuilderTests.java   |  1 -
 .../query/WildcardQueryBuilderTests.java      |  3 +-
 .../FunctionScoreQueryBuilderTests.java       | 18 ----
 .../bucket/histogram/DateHistogramTests.java  |  5 -
 .../test/AbstractBuilderTestCase.java         | 97 +++++++------------
 .../test/AbstractQueryTestCase.java           |  9 +-
 26 files changed, 105 insertions(+), 336 deletions(-)

diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/FeatureQueryBuilderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/FeatureQueryBuilderTests.java
index dd7b42487fd..40da4b53227 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/FeatureQueryBuilderTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/FeatureQueryBuilderTests.java
@@ -45,12 +45,10 @@ public class FeatureQueryBuilderTests extends AbstractQueryTestCase<FeatureQuery
 
     @Override
     protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
-        for (String type : getCurrentTypes()) {
-            mapperService.merge(type, new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef(type,
-                    "my_feature_field", "type=feature",
-                    "my_negative_feature_field", "type=feature,positive_score_impact=false",
-                    "my_feature_vector_field", "type=feature_vector"))), MapperService.MergeReason.MAPPING_UPDATE);
-        }
+        mapperService.merge("_doc", new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef("_doc",
+            "my_feature_field", "type=feature",
+            "my_negative_feature_field", "type=feature,positive_score_impact=false",
+            "my_feature_vector_field", "type=feature_vector"))), MapperService.MergeReason.MAPPING_UPDATE);
     }
 
     @Override
@@ -87,7 +85,7 @@ public class FeatureQueryBuilderTests extends AbstractQueryTestCase<FeatureQuery
         if (mayUseNegativeField) {
             fields.add("my_negative_feature_field");
         }
-        
+
         final String field = randomFrom(fields);
         return new FeatureQueryBuilder(field, function);
     }
@@ -99,7 +97,6 @@ public class FeatureQueryBuilderTests extends AbstractQueryTestCase<FeatureQuery
     }
 
     public void testDefaultScoreFunction() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"feature\" : {\n" +
                 "        \"field\": \"my_feature_field\"\n" +
@@ -110,7 +107,6 @@ public class FeatureQueryBuilderTests extends AbstractQueryTestCase<FeatureQuery
     }
 
     public void testIllegalField() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"feature\" : {\n" +
                 "        \"field\": \"" + STRING_FIELD_NAME + "\"\n" +
@@ -121,7 +117,6 @@ public class FeatureQueryBuilderTests extends AbstractQueryTestCase<FeatureQuery
     }
 
     public void testIllegalCombination() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"feature\" : {\n" +
                 "        \"field\": \"my_negative_feature_field\",\n" +
diff --git a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
index ddd3ddf486f..98a5d91e1b1 100644
--- a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
@@ -89,7 +89,6 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase<DisMaxQueryBu
     }
 
     public void testToQueryInnerPrefixQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String queryAsString = "{\n" +
                 "    \"dis_max\":{\n" +
                 "        \"queries\":[\n" +
diff --git a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
index 88742e08554..ad02209bf5d 100644
--- a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
@@ -68,11 +68,7 @@ public class ExistsQueryBuilderTests extends AbstractQueryTestCase<ExistsQueryBu
         Collection<String> fields = context.getQueryShardContext().simpleMatchToIndexNames(fieldPattern);
         Collection<String> mappedFields = fields.stream().filter((field) -> context.getQueryShardContext().getObjectMapper(field) != null
                 || context.getQueryShardContext().getMapperService().fullName(field) != null).collect(Collectors.toList());
-        if (getCurrentTypes().length == 0) {
-            assertThat(query, instanceOf(MatchNoDocsQuery.class));
-            MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query;
-            assertThat(matchNoDocsQuery.toString(null), containsString("Missing types in \"exists\" query."));
-        } else if (context.mapperService().getIndexSettings().getIndexVersionCreated().before(Version.V_6_1_0)) {
+        if (context.mapperService().getIndexSettings().getIndexVersionCreated().before(Version.V_6_1_0)) {
             if (fields.size() == 1) {
                 assertThat(query, instanceOf(ConstantScoreQuery.class));
                 ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query;
diff --git a/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
index 0a67ab50ae8..0a0d567c7a4 100644
--- a/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
@@ -105,7 +105,6 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
     }
 
     public void testToQueryWithStringField() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"fuzzy\":{\n" +
                 "        \"" + STRING_FIELD_NAME + "\":{\n" +
@@ -128,7 +127,6 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
     }
 
     public void testToQueryWithStringFieldDefinedFuzziness() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
             "    \"fuzzy\":{\n" +
             "        \"" + STRING_FIELD_NAME + "\":{\n" +
@@ -151,7 +149,6 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
     }
 
     public void testToQueryWithStringFieldDefinedWrongFuzziness() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String queryMissingFuzzinessUpLimit = "{\n" +
             "    \"fuzzy\":{\n" +
             "        \"" + STRING_FIELD_NAME + "\":{\n" +
@@ -214,7 +211,6 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
     }
 
     public void testToQueryWithNumericField() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"fuzzy\":{\n" +
                 "        \"" + INT_FIELD_NAME + "\":{\n" +
@@ -299,7 +295,6 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
     }
 
     public void testToQueryWithTranspositions() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = new FuzzyQueryBuilder(STRING_FIELD_NAME, "text").toQuery(createShardContext());
         assertThat(query, instanceOf(FuzzyQuery.class));
         assertEquals(FuzzyQuery.defaultTranspositions, ((FuzzyQuery)query).getTranspositions());
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
index d0d075af21a..551bd6488cd 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
@@ -39,7 +39,6 @@ import java.io.IOException;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.Matchers.startsWith;
 
 public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBoundingBoxQueryBuilder> {
     /** Randomly generate either NaN or one of the two infinity values. */
@@ -110,16 +109,12 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
         assertEquals("cannot parse type from null string", e.getMessage());
     }
 
-    @Override
-    public void testToQuery() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testToQuery();
-    }
-
-    public void testExceptionOnMissingTypes() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length == 0);
-        QueryShardException e = expectThrows(QueryShardException.class, super::testToQuery);
-        assertThat(e.getMessage(), startsWith("failed to find geo_point field [mapped_geo_point"));
+    public void testExceptionOnMissingTypes() {
+        QueryShardContext context = createShardContextWithNoType();
+        GeoBoundingBoxQueryBuilder qb = createTestQueryBuilder();
+        qb.ignoreUnmapped(false);
+        QueryShardException e = expectThrows(QueryShardException.class, () -> qb.toQuery(context));
+        assertEquals("failed to find geo_point field [" + qb.fieldName() + "]", e.getMessage());
     }
 
     public void testBrokenCoordinateCannotBeSet() {
@@ -295,7 +290,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
     }
 
     public void testParsingAndToQuery1() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_bounding_box\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME+ "\":{\n" +
@@ -308,7 +302,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
     }
 
     public void testParsingAndToQuery2() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_bounding_box\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME+ "\":{\n" +
@@ -327,7 +320,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
     }
 
     public void testParsingAndToQuery3() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_bounding_box\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME+ "\":{\n" +
@@ -340,7 +332,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
     }
 
     public void testParsingAndToQuery4() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_bounding_box\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME+ "\":{\n" +
@@ -353,7 +344,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
     }
 
     public void testParsingAndToQuery5() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_bounding_box\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME+ "\":{\n" +
@@ -366,7 +356,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
     }
 
     public void testParsingAndToQuery6() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_bounding_box\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME+ "\":{\n" +
@@ -513,7 +502,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
     }
 
     public void testHonorsCoercion() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
             "  \"geo_bounding_box\": {\n" +
             "    \"validation_method\": \"COERCE\",\n" +
@@ -534,7 +522,6 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
 
     @Override
     public void testMustRewrite() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         super.testMustRewrite();
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
index 7b7a1dcb7a5..0cb6a74570b 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
@@ -122,7 +122,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
      */
     @Override
     public void testToQuery() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         super.testToQuery();
     }
 
@@ -148,7 +147,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery1() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":\"12mi\",\n" +
@@ -162,7 +160,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery2() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":\"12mi\",\n" +
@@ -173,7 +170,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery3() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":\"12mi\",\n" +
@@ -184,7 +180,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery4() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":\"12mi\",\n" +
@@ -195,7 +190,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery5() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":12,\n" +
@@ -210,7 +204,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery6() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":\"12\",\n" +
@@ -225,7 +218,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery7() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "  \"geo_distance\":{\n" +
                 "      \"distance\":\"19.312128\",\n" +
@@ -239,7 +231,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery8() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":19.312128,\n" +
@@ -253,7 +244,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery9() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":\"19.312128\",\n" +
@@ -268,7 +258,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery10() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":19.312128,\n" +
@@ -283,7 +272,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery11() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":\"19.312128km\",\n" +
@@ -297,7 +285,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     public void testParsingAndToQuery12() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_distance\":{\n" +
                 "        \"distance\":\"12mi\",\n" +
@@ -312,7 +299,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
     }
 
     private void assertGeoDistanceRangeQuery(String query, double lat, double lon, double distance, DistanceUnit distanceUnit) throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query parsedQuery = parseQuery(query).toQuery(createShardContext());
         // TODO: what can we check?
     }
@@ -336,12 +322,6 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
         assertEquals(json, 12000.0, parsed.distance(), 0.0001);
     }
 
-    @Override
-    public void testMustRewrite() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testMustRewrite();
-    }
-
     public void testIgnoreUnmapped() throws IOException {
         final GeoDistanceQueryBuilder queryBuilder = new GeoDistanceQueryBuilder("unmapped").point(0.0, 0.0).distance("20m");
         queryBuilder.ignoreUnmapped(true);
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
index 8f1ae8eee07..ee64e595e08 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
@@ -62,17 +62,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
         // todo LatLonPointInPolygon is package private
     }
 
-    /**
-     * Overridden here to ensure the test is only run if at least one type is
-     * present in the mappings. Geo queries do not execute if the field is not
-     * explicitly mapped
-     */
-    @Override
-    public void testToQuery() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testToQuery();
-    }
-
     private static List<GeoPoint> randomPolygon() {
         ShapeBuilder<?, ?> shapeBuilder = null;
         // This is a temporary fix because sometimes the RandomShapeGenerator
@@ -138,7 +127,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
     }
 
     public void testParsingAndToQuery1() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_polygon\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME + "\":{\n" +
@@ -154,7 +142,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
     }
 
     public void testParsingAndToQuery2() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_polygon\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME + "\":{\n" +
@@ -179,7 +166,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
     }
 
     public void testParsingAndToQuery3() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_polygon\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME + "\":{\n" +
@@ -195,7 +181,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
     }
 
     public void testParsingAndToQuery4() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"geo_polygon\":{\n" +
                 "        \"" + GEO_POINT_FIELD_NAME + "\":{\n" +
@@ -234,12 +219,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
         assertEquals(json, 4, parsed.points().size());
     }
 
-    @Override
-    public void testMustRewrite() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testMustRewrite();
-    }
-
     public void testIgnoreUnmapped() throws IOException {
         List<GeoPoint> polygon = randomPolygon();
         final GeoPolygonQueryBuilder queryBuilder = new GeoPolygonQueryBuilder("unmapped", polygon);
@@ -255,7 +234,6 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
     }
 
     public void testPointValidation() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext context = createShardContext();
         String queryInvalidLat = "{\n" +
             "    \"geo_polygon\":{\n" +
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
index eafb4995f72..ca9a21973aa 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
@@ -153,17 +153,6 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase<GeoShapeQue
         assertThat(query, anyOf(instanceOf(BooleanQuery.class), instanceOf(ConstantScoreQuery.class)));
     }
 
-    /**
-     * Overridden here to ensure the test is only run if at least one type is
-     * present in the mappings. Geo queries do not execute if the field is not
-     * explicitly mapped
-     */
-    @Override
-    public void testToQuery() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testToQuery();
-    }
-
     public void testNoFieldName() throws Exception {
         ShapeBuilder<?, ?> shape = RandomShapeGenerator.createShapeWithin(random(), null);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new GeoShapeQueryBuilder(null, shape));
@@ -279,7 +268,6 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase<GeoShapeQue
     }
 
     public void testWrongFieldType() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         ShapeType shapeType = ShapeType.randomType(random());
         ShapeBuilder<?, ?> shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType);
         final GeoShapeQueryBuilder queryBuilder = new GeoShapeQueryBuilder(STRING_FIELD_NAME, shape);
diff --git a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
index 727821039dc..c146df73019 100644
--- a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
@@ -40,23 +40,17 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder>
 
     @Override
     protected IdsQueryBuilder doCreateTestQueryBuilder() {
-        String[] types;
-        if (getCurrentTypes() != null && getCurrentTypes().length > 0 && randomBoolean()) {
-            int numberOfTypes = randomIntBetween(1, getCurrentTypes().length);
-            types = new String[numberOfTypes];
-            for (int i = 0; i < numberOfTypes; i++) {
-                if (frequently()) {
-                    types[i] = randomFrom(getCurrentTypes());
-                } else {
-                    types[i] = randomAlphaOfLengthBetween(1, 10);
-                }
-            }
-        } else {
-            if (randomBoolean()) {
-                types = new String[]{MetaData.ALL};
+        final String type;
+        if (randomBoolean()) {
+            if (frequently()) {
+                type = "_doc";
             } else {
-                types = new String[0];
+                type = randomAlphaOfLengthBetween(1, 10);
             }
+        } else if (randomBoolean()) {
+                type = MetaData.ALL;
+        } else {
+            type = null;
         }
         int numberOfIds = randomIntBetween(0, 10);
         String[] ids = new String[numberOfIds];
@@ -64,8 +58,8 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder>
             ids[i] = randomAlphaOfLengthBetween(1, 10);
         }
         IdsQueryBuilder query;
-        if (types.length > 0 || randomBoolean()) {
-            query = new IdsQueryBuilder().types(types);
+        if (type != null && randomBoolean()) {
+            query = new IdsQueryBuilder().types(type);
             query.addIds(ids);
         } else {
             query = new IdsQueryBuilder();
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
index 694b63b141b..fd722ef0c77 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
@@ -34,6 +34,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.test.AbstractBuilderTestCase.STRING_ALIAS_FIELD_NAME;
 import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.containsString;
@@ -42,11 +43,8 @@ import static org.hamcrest.Matchers.notNullValue;
 public class MatchPhrasePrefixQueryBuilderTests extends AbstractQueryTestCase<MatchPhrasePrefixQueryBuilder> {
     @Override
     protected MatchPhrasePrefixQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME,
-            INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
-        if (fieldName.equals(DATE_FIELD_NAME)) {
-            assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        }
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME,
+                DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
         Object value;
         if (isTextField(fieldName)) {
             int terms = randomIntBetween(0, 3);
@@ -119,7 +117,6 @@ public class MatchPhrasePrefixQueryBuilderTests extends AbstractQueryTestCase<Ma
     }
 
     public void testPhraseOnFieldWithNoTerms() {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         MatchPhrasePrefixQueryBuilder matchQuery = new MatchPhrasePrefixQueryBuilder(DATE_FIELD_NAME, "three term phrase");
         matchQuery.analyzer("whitespace");
         expectThrows(IllegalArgumentException.class, () -> matchQuery.doToQuery(createShardContext()));
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
index e59c5d6e0c4..14ef7940467 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
@@ -36,6 +36,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.test.AbstractBuilderTestCase.STRING_ALIAS_FIELD_NAME;
 import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.containsString;
@@ -45,11 +46,8 @@ import static org.hamcrest.Matchers.notNullValue;
 public class MatchPhraseQueryBuilderTests extends AbstractQueryTestCase<MatchPhraseQueryBuilder> {
     @Override
     protected MatchPhraseQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME,
-            INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
-        if (fieldName.equals(DATE_FIELD_NAME)) {
-            assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        }
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME,
+                DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
         Object value;
         if (isTextField(fieldName)) {
             int terms = randomIntBetween(0, 3);
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index cdaccb486f2..0de9cac8855 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -61,11 +61,8 @@ import static org.hamcrest.Matchers.notNullValue;
 public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuilder> {
     @Override
     protected MatchQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME,
-            INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
-        if (fieldName.equals(DATE_FIELD_NAME)) {
-            assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        }
+        String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME,
+                DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
         Object value;
         if (isTextField(fieldName)) {
             int terms = randomIntBetween(0, 3);
@@ -279,7 +276,6 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
     }
 
     public void testFuzzinessOnNonStringField() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         MatchQueryBuilder query = new MatchQueryBuilder(INT_FIELD_NAME, 42);
         query.fuzziness(randomFuzziness(INT_FIELD_NAME));
         QueryShardContext context = createShardContext();
@@ -300,7 +296,6 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
     }
 
     public void testExactOnUnsupportedField() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         MatchQueryBuilder query = new MatchQueryBuilder(GEO_POINT_FIELD_NAME, "2,3");
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class, () -> query.toQuery(context));
@@ -352,7 +347,6 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
     }
 
     public void testExceptionUsingAnalyzerOnNumericField() {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext shardContext = createShardContext();
         MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(DOUBLE_FIELD_NAME, 6.075210893508043E-4);
         matchQueryBuilder.analyzer("simple");
@@ -371,7 +365,6 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
     }
 
     public void testMatchPhrasePrefixWithBoost() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext context = createShardContext();
         assumeTrue("test runs only when the index version is on or after V_5_0_0_alpha1",
             context.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1));
@@ -395,7 +388,6 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
     }
 
     public void testLenientPhraseQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext context = createShardContext();
         MatchQuery b = new MatchQuery(context);
         b.setLenient(true);
diff --git a/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index b54ce571453..6ac97373dfa 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -294,7 +294,6 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase<MoreLik
     }
 
     public void testUnsupportedFields() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String unsupportedField = randomFrom(INT_FIELD_NAME, DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
         MoreLikeThisQueryBuilder queryBuilder = new MoreLikeThisQueryBuilder(new String[] {unsupportedField}, new String[]{"some text"}, null)
                 .failOnUnsupportedField(true);
diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
index 9eed4d9f1c1..e30cdaca402 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
@@ -65,9 +65,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     protected MultiMatchQueryBuilder doCreateTestQueryBuilder() {
         String fieldName = randomFrom(STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME,
                 MISSING_FIELD_NAME, MISSING_WILDCARD_FIELD_NAME);
-        if (fieldName.equals(DATE_FIELD_NAME)) {
-            assumeTrue("test with date fields runs only when at least a type is registered", getCurrentTypes().length > 0);
-        }
 
         final Object value;
         if (fieldName.equals(STRING_FIELD_NAME)) {
@@ -173,7 +170,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testToQueryBoost() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext shardContext = createShardContext();
         MultiMatchQueryBuilder multiMatchQueryBuilder = new MultiMatchQueryBuilder("test");
         multiMatchQueryBuilder.field(STRING_FIELD_NAME, 5f);
@@ -191,7 +187,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testToQueryMultipleTermsBooleanQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = multiMatchQuery("test1 test2").field(STRING_FIELD_NAME).useDisMax(false).toQuery(createShardContext());
         assertThat(query, instanceOf(BooleanQuery.class));
         BooleanQuery bQuery = (BooleanQuery) query;
@@ -201,7 +196,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testToQueryMultipleFieldsDisableDismax() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = multiMatchQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(false).toQuery(createShardContext());
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
@@ -212,7 +206,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testToQueryMultipleFieldsDisMaxQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = multiMatchQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(true).toQuery(createShardContext());
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query;
@@ -225,7 +218,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testToQueryFieldsWildcard() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = multiMatchQuery("test").field("mapped_str*").useDisMax(false).toQuery(createShardContext());
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
@@ -237,7 +229,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testToQueryFieldMissing() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         assertThat(multiMatchQuery("test").field(MISSING_WILDCARD_FIELD_NAME).toQuery(createShardContext()), instanceOf(MatchNoDocsQuery.class));
         assertThat(multiMatchQuery("test").field(MISSING_FIELD_NAME).toQuery(createShardContext()), instanceOf(MatchNoDocsQuery.class));
     }
@@ -307,7 +298,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testExceptionUsingAnalyzerOnNumericField() {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext shardContext = createShardContext();
         MultiMatchQueryBuilder multiMatchQueryBuilder = new MultiMatchQueryBuilder(6.075210893508043E-4);
         multiMatchQueryBuilder.field(DOUBLE_FIELD_NAME);
@@ -317,7 +307,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testFuzzinessOnNonStringField() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         MultiMatchQueryBuilder query = new MultiMatchQueryBuilder(42).field(INT_FIELD_NAME).field(BOOLEAN_FIELD_NAME);
         query.fuzziness(randomFuzziness(INT_FIELD_NAME));
         QueryShardContext context = createShardContext();
@@ -336,8 +325,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testToFuzzyQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-
         MultiMatchQueryBuilder qb = new MultiMatchQueryBuilder("text").field(STRING_FIELD_NAME);
         qb.fuzziness(Fuzziness.TWO);
         qb.prefixLength(2);
@@ -351,7 +338,6 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     }
 
     public void testDefaultField() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext context = createShardContext();
         MultiMatchQueryBuilder builder = new MultiMatchQueryBuilder("hello");
         // should pass because we set lenient to true when default field is `*`
diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index c78c83c154f..2f868d02921 100644
--- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -109,7 +109,6 @@ public class PrefixQueryBuilderTests extends AbstractQueryTestCase<PrefixQueryBu
     }
 
     public void testNumeric() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         PrefixQueryBuilder query = prefixQuery(INT_FIELD_NAME, "12*");
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class,
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index 591ee5af080..87197b662d1 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -401,7 +401,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryTermQuery() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("test").defaultField(STRING_FIELD_NAME).toQuery(createShardContext());
         assertThat(query, instanceOf(TermQuery.class));
         TermQuery termQuery = (TermQuery) query;
@@ -409,7 +408,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryPhraseQuery() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("\"term1 term2\"")
             .defaultField(STRING_FIELD_NAME)
             .phraseSlop(3)
@@ -423,7 +421,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryBoosts() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext shardContext = createShardContext();
         QueryStringQueryBuilder queryStringQuery = queryStringQuery(STRING_FIELD_NAME + ":boosted^2");
         Query query = queryStringQuery.toQuery(shardContext);
@@ -463,7 +460,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryMultipleTermsBooleanQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("test1 test2").field(STRING_FIELD_NAME)
             .toQuery(createShardContext());
         assertThat(query, instanceOf(BooleanQuery.class));
@@ -476,7 +472,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryMultipleFieldsBooleanQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("test").field(STRING_FIELD_NAME)
             .field(STRING_FIELD_NAME_2)
             .toQuery(createShardContext());
@@ -490,7 +485,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryMultipleFieldsDisMaxQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2)
             .toQuery(createShardContext());
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
@@ -501,7 +495,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryFieldsWildcard() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("test").field("mapped_str*").toQuery(createShardContext());
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
@@ -515,7 +508,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryDisMaxQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("test").field(STRING_FIELD_NAME, 2.2f)
             .field(STRING_FIELD_NAME_2)
             .toQuery(createShardContext());
@@ -527,7 +519,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryWildcardQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         for (Operator op : Operator.values()) {
             BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
             QueryStringQueryParser queryParser = new QueryStringQueryParser(createShardContext(), STRING_FIELD_NAME);
@@ -550,7 +541,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryWilcardQueryWithSynonyms() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         for (Operator op : Operator.values()) {
             BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
             QueryStringQueryParser queryParser = new QueryStringQueryParser(createShardContext(), STRING_FIELD_NAME);
@@ -583,7 +573,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryWithGraph() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         for (Operator op : Operator.values()) {
             BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
             QueryStringQueryParser queryParser = new QueryStringQueryParser(createShardContext(), STRING_FIELD_NAME);
@@ -698,7 +687,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryRegExpQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("/foo*bar/").defaultField(STRING_FIELD_NAME)
             .maxDeterminizedStates(5000)
             .toQuery(createShardContext());
@@ -708,7 +696,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryRegExpQueryTooComplex() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryStringQueryBuilder queryBuilder = queryStringQuery("/[ac]*a[ac]{50,200}/").defaultField(STRING_FIELD_NAME);
 
         TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class,
@@ -721,7 +708,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
      * Validates that {@code max_determinized_states} can be parsed and lowers the allowed number of determinized states.
      */
     public void testToQueryRegExpQueryMaxDeterminizedStatesParsing() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject(); {
             builder.startObject("query_string"); {
@@ -744,7 +730,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
      * Validates that {@code max_determinized_states} can be parsed and lowers the allowed number of determinized states.
      */
     public void testEnabledPositionIncrements() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
 
         XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject(); {
@@ -762,8 +747,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryFuzzyQueryAutoFuziness() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-
         int length = randomIntBetween(1, 10);
         StringBuilder queryString = new StringBuilder();
         for (int i = 0; i < length; i++) {
@@ -788,7 +771,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testFuzzyNumeric() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryStringQueryBuilder query = queryStringQuery("12~0.2").defaultField(INT_FIELD_NAME);
         QueryShardContext context = createShardContext();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
@@ -800,7 +782,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testPrefixNumeric() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryStringQueryBuilder query = queryStringQuery("12*").defaultField(INT_FIELD_NAME);
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class,
@@ -812,7 +793,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testExactGeo() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryStringQueryBuilder query = queryStringQuery("2,3").defaultField(GEO_POINT_FIELD_NAME);
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class,
@@ -824,7 +804,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testTimezone() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String queryAsString = "{\n" +
                 "    \"query_string\":{\n" +
                 "        \"time_zone\":\"Europe/Paris\",\n" +
@@ -846,7 +825,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryBooleanQueryMultipleBoosts() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         int numBoosts = randomIntBetween(2, 10);
         float[] boosts = new float[numBoosts + 1];
         String queryStringPrefix = "";
@@ -885,7 +863,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryPhraseQueryBoostAndSlop() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryStringQueryBuilder queryStringQueryBuilder =
             new QueryStringQueryBuilder("\"test phrase\"~2").field(STRING_FIELD_NAME, 5f);
         Query query = queryStringQueryBuilder.toQuery(createShardContext());
@@ -899,7 +876,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryWildcardNonExistingFields() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryStringQueryBuilder queryStringQueryBuilder =
             new QueryStringQueryBuilder("foo bar").field("invalid*");
         Query query = queryStringQueryBuilder.toQuery(createShardContext());
@@ -918,7 +894,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToQueryTextParsing() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         {
             QueryStringQueryBuilder queryBuilder =
                 new QueryStringQueryBuilder("foo bar")
@@ -1017,16 +992,15 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         QueryShardContext context = createShardContext();
         QueryStringQueryBuilder queryBuilder = new QueryStringQueryBuilder(STRING_FIELD_NAME + ":*");
         Query query = queryBuilder.toQuery(context);
-        if (getCurrentTypes().length > 0) {
-            if (context.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
-                    && (context.fieldMapper(STRING_FIELD_NAME).omitNorms() == false)) {
-                assertThat(query, equalTo(new ConstantScoreQuery(new NormsFieldExistsQuery(STRING_FIELD_NAME))));
-            } else {
-                assertThat(query, equalTo(new ConstantScoreQuery(new TermQuery(new Term("_field_names", STRING_FIELD_NAME)))));
-            }
+        if (context.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
+                && (context.fieldMapper(STRING_FIELD_NAME).omitNorms() == false)) {
+            assertThat(query, equalTo(new ConstantScoreQuery(new NormsFieldExistsQuery(STRING_FIELD_NAME))));
         } else {
-            assertThat(query, equalTo(new MatchNoDocsQuery()));
+            assertThat(query, equalTo(new ConstantScoreQuery(new TermQuery(new Term("_field_names", STRING_FIELD_NAME)))));
         }
+        QueryShardContext contextNoType = createShardContextWithNoType();
+        query = queryBuilder.toQuery(contextNoType);
+        assertThat(query, equalTo(new MatchNoDocsQuery()));
 
         queryBuilder = new QueryStringQueryBuilder("*:*");
         query = queryBuilder.toQuery(context);
@@ -1040,7 +1014,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testDisabledFieldNamesField() throws Exception {
-        assumeTrue("No types", getCurrentTypes().length > 0);
         QueryShardContext context = createShardContext();
         context.getMapperService().merge("_doc",
             new CompressedXContent(
@@ -1098,7 +1071,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testExpandedTerms() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         // Prefix
         Query query = new QueryStringQueryBuilder("aBc*")
                 .field(STRING_FIELD_NAME)
@@ -1161,7 +1133,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testLenientRewriteToMatchNoDocs() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         // Term
         Query query = new QueryStringQueryBuilder("hello")
             .field(INT_FIELD_NAME)
@@ -1207,7 +1178,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testDefaultField() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext context = createShardContext();
         context.getIndexSettings().updateIndexMetaData(
             newIndexMeta("index", context.getIndexSettings().getSettings(), Settings.builder().putList("index.query.default_field",
@@ -1233,7 +1203,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
      * the quote analyzer should overwrite any other forced analyzer in quoted parts of the query
      */
     public void testQuoteAnalyzer() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         // Prefix
         Query query = new QueryStringQueryBuilder("ONE \"TWO THREE\"")
                 .field(STRING_FIELD_NAME)
@@ -1252,7 +1221,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testQuoteFieldSuffix() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext context = createShardContext();
         assertEquals(new TermQuery(new Term(STRING_FIELD_NAME, "bar")),
             new QueryStringQueryBuilder("bar")
@@ -1283,8 +1251,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testToFuzzyQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-
         Query query = new QueryStringQueryBuilder("text~2")
             .field(STRING_FIELD_NAME)
             .fuzzyPrefixLength(2)
@@ -1296,7 +1262,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testWithStopWords() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = new QueryStringQueryBuilder("the quick fox")
             .field(STRING_FIELD_NAME)
             .analyzer("stop")
@@ -1309,7 +1274,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
     }
 
     public void testWithPrefixStopWords() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = new QueryStringQueryBuilder("the* quick fox")
             .field(STRING_FIELD_NAME)
             .analyzer("stop")
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index ba02ddaf05c..6be12cc841a 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -133,26 +133,20 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
         String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
         if (queryBuilder.from() == null && queryBuilder.to() == null) {
             final Query expectedQuery;
-            if (getCurrentTypes().length > 0) {
-                if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
-                        && context.mapperService().fullName(queryBuilder.fieldName()).hasDocValues()) {
-                    expectedQuery = new ConstantScoreQuery(new DocValuesFieldExistsQuery(expectedFieldName));
-                } else if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
-                        && context.mapperService().fullName(queryBuilder.fieldName()).omitNorms() == false) {
-                    expectedQuery = new ConstantScoreQuery(new NormsFieldExistsQuery(expectedFieldName));
-                } else {
-                    expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, expectedFieldName)));
-                }
+            if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
+                    && context.mapperService().fullName(queryBuilder.fieldName()).hasDocValues()) {
+                expectedQuery = new ConstantScoreQuery(new DocValuesFieldExistsQuery(expectedFieldName));
+            } else if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0) &&
+                            context.mapperService().fullName(queryBuilder.fieldName()).omitNorms() == false) {
+                expectedQuery = new ConstantScoreQuery(new NormsFieldExistsQuery(expectedFieldName));
             } else {
-                expectedQuery = new MatchNoDocsQuery("no mappings yet");
+                expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, expectedFieldName)));
             }
             assertThat(query, equalTo(expectedQuery));
-
-        } else if (getCurrentTypes().length == 0 ||
-            (expectedFieldName.equals(DATE_FIELD_NAME) == false
-                && expectedFieldName.equals(INT_FIELD_NAME) == false
-                && expectedFieldName.equals(DATE_RANGE_FIELD_NAME) == false
-                && expectedFieldName.equals(INT_RANGE_FIELD_NAME) == false)) {
+        } else if (expectedFieldName.equals(DATE_FIELD_NAME) == false &&
+                        expectedFieldName.equals(INT_FIELD_NAME) == false &&
+                        expectedFieldName.equals(DATE_RANGE_FIELD_NAME) == false &&
+                        expectedFieldName.equals(INT_RANGE_FIELD_NAME) == false) {
             assertThat(query, instanceOf(TermRangeQuery.class));
             TermRangeQuery termRangeQuery = (TermRangeQuery) query;
             assertThat(termRangeQuery.getField(), equalTo(expectedFieldName));
@@ -165,7 +159,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
             query = ((IndexOrDocValuesQuery) query).getIndexQuery();
             assertThat(query, instanceOf(PointRangeQuery.class));
             MapperService mapperService = context.getQueryShardContext().getMapperService();
-            MappedFieldType mappedFieldType = mapperService.fullName(DATE_FIELD_NAME);
+            MappedFieldType mappedFieldType = mapperService.fullName(expectedFieldName);
             final Long fromInMillis;
             final Long toInMillis;
             // we have to normalize the incoming value into milliseconds since it could be literally anything
@@ -257,7 +251,6 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
     }
 
     public void testToQueryNumericField() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query parsedQuery = rangeQuery(INT_FIELD_NAME).from(23).to(54).includeLower(true).includeUpper(false).toQuery(createShardContext());
         // since age is automatically registered in data, we encode it as numeric
         assertThat(parsedQuery, instanceOf(IndexOrDocValuesQuery.class));
@@ -267,7 +260,6 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
     }
 
     public void testDateRangeQueryFormat() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         // We test 01/01/2012 from gte and 2030 for lt
         String query = "{\n" +
                 "    \"range\" : {\n" +
@@ -302,7 +294,6 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
     }
 
     public void testDateRangeBoundaries() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"range\" : {\n" +
                 "        \"" + DATE_FIELD_NAME + "\" : {\n" +
@@ -339,7 +330,6 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
     }
 
     public void testDateRangeQueryTimezone() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String query = "{\n" +
                 "    \"range\" : {\n" +
                 "        \"" + DATE_FIELD_NAME + "\" : {\n" +
@@ -427,19 +417,19 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
         assertThat(rewrittenRange.to(), equalTo(null));
 
         // Range query with open bounds rewrite to an exists query
-        final Query luceneQuery = rewrittenRange.toQuery(queryShardContext);
+        Query luceneQuery = rewrittenRange.toQuery(queryShardContext);
         final Query expectedQuery;
-        if (getCurrentTypes().length > 0) {
-            if (queryShardContext.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
-                    && queryShardContext.fieldMapper(query.fieldName()).hasDocValues()) {
-                expectedQuery = new ConstantScoreQuery(new DocValuesFieldExistsQuery(query.fieldName()));
-            } else {
-                expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, query.fieldName())));
-            }
+        if (queryShardContext.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)
+                && queryShardContext.fieldMapper(query.fieldName()).hasDocValues()) {
+            expectedQuery = new ConstantScoreQuery(new DocValuesFieldExistsQuery(query.fieldName()));
         } else {
-            expectedQuery = new MatchNoDocsQuery("no mappings yet");
+            expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, query.fieldName())));
         }
         assertThat(luceneQuery, equalTo(expectedQuery));
+
+        QueryShardContext queryShardContextWithUnkType = createShardContextWithNoType();
+        luceneQuery  = rewrittenRange.toQuery(queryShardContextWithUnkType);
+        assertThat(luceneQuery, equalTo(new MatchNoDocsQuery("no mappings yet")));
     }
 
     public void testRewriteDateToMatchAllWithTimezoneAndFormat() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
index 1110388090a..4ba6165c061 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
@@ -116,7 +116,6 @@ public class RegexpQueryBuilderTests extends AbstractQueryTestCase<RegexpQueryBu
     }
 
     public void testNumeric() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         RegexpQueryBuilder query = new RegexpQueryBuilder(INT_FIELD_NAME, "12");
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class, () -> query.toQuery(context));
diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index 3c508ddc8be..a2d6e3ab361 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -247,11 +247,8 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
         assertThat(queryBuilder.fields().size(), equalTo(0));
         QueryShardContext shardContext = createShardContext();
 
-        // the remaining tests requires either a mapping that we register with types in base test setup
-        if (getCurrentTypes().length > 0) {
-            Query luceneQuery = queryBuilder.toQuery(shardContext);
-            assertThat(luceneQuery, anyOf(instanceOf(BooleanQuery.class), instanceOf(DisjunctionMaxQuery.class)));
-        }
+        Query luceneQuery = queryBuilder.toQuery(shardContext);
+        assertThat(luceneQuery, anyOf(instanceOf(BooleanQuery.class), instanceOf(DisjunctionMaxQuery.class)));
     }
 
     /*
@@ -315,7 +312,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testToQueryBoost() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext shardContext = createShardContext();
         SimpleQueryStringBuilder simpleQueryStringBuilder = new SimpleQueryStringBuilder("test");
         simpleQueryStringBuilder.field(STRING_FIELD_NAME, 5);
@@ -380,7 +376,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testMinimumShouldMatch() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext shardContext = createShardContext();
         int numberOfTerms = randomIntBetween(1, 4);
         StringBuilder queryString = new StringBuilder();
@@ -421,13 +416,10 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
         simpleQueryStringBuilder.field("_index");
         Query query = simpleQueryStringBuilder.toQuery(shardContext);
         assertThat(query, notNullValue());
-        if (getCurrentTypes().length > 0) {
-            assertThat(query, instanceOf(MatchAllDocsQuery.class));
-        }
+        assertThat(query, instanceOf(MatchAllDocsQuery.class));
     }
 
     public void testExpandedTerms() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         // Prefix
         Query query = new SimpleQueryStringBuilder("aBc*")
                 .field(STRING_FIELD_NAME)
@@ -456,7 +448,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testAnalyzeWildcard() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         SimpleQueryStringQueryParser.Settings settings = new SimpleQueryStringQueryParser.Settings();
         settings.analyzeWildcard(true);
         SimpleQueryStringQueryParser parser = new SimpleQueryStringQueryParser(new StandardAnalyzer(),
@@ -480,7 +471,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testAnalyzerWildcardWithSynonyms() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         SimpleQueryStringQueryParser.Settings settings = new SimpleQueryStringQueryParser.Settings();
         settings.analyzeWildcard(true);
         SimpleQueryStringQueryParser parser = new SimpleQueryStringQueryParser(new MockRepeatAnalyzer(),
@@ -512,7 +502,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testAnalyzerWithGraph() {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         SimpleQueryStringQueryParser.Settings settings = new SimpleQueryStringQueryParser.Settings();
         settings.analyzeWildcard(true);
         SimpleQueryStringQueryParser parser = new SimpleQueryStringQueryParser(new MockSynonymAnalyzer(),
@@ -557,7 +546,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testQuoteFieldSuffix() {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         SimpleQueryStringQueryParser.Settings settings = new SimpleQueryStringQueryParser.Settings();
         settings.analyzeWildcard(true);
         settings.quoteFieldSuffix("_2");
@@ -575,7 +563,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testDefaultField() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         QueryShardContext context = createShardContext();
         context.getIndexSettings().updateIndexMetaData(
             newIndexMeta("index", context.getIndexSettings().getSettings(), Settings.builder().putList("index.query.default_field",
@@ -598,8 +585,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testToFuzzyQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-
         Query query = new SimpleQueryStringBuilder("text~2")
             .field(STRING_FIELD_NAME)
             .fuzzyPrefixLength(2)
@@ -611,8 +596,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testLenientToPrefixQuery() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-
         Query query = new SimpleQueryStringBuilder("t*")
             .field(DATE_FIELD_NAME)
             .field(STRING_FIELD_NAME)
@@ -626,7 +609,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testWithStopWords() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = new SimpleQueryStringBuilder("the quick fox")
             .field(STRING_FIELD_NAME)
             .analyzer("stop")
@@ -639,7 +621,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
     }
 
     public void testWithPrefixStopWords() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = new SimpleQueryStringBuilder("the* quick fox")
             .field(STRING_FIELD_NAME)
             .analyzer("stop")
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
index b6cc150717a..6876d021a0a 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
@@ -138,7 +138,6 @@ public class TermQueryBuilderTests extends AbstractTermQueryTestCase<TermQueryBu
     }
 
     public void testGeo() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         TermQueryBuilder query = new TermQueryBuilder(GEO_POINT_FIELD_NAME, "2,3");
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class, () -> query.toQuery(context));
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index 963a08ce753..9ed303316d9 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -264,7 +264,6 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
     }
 
     public void testGeo() throws Exception {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         TermsQueryBuilder query = new TermsQueryBuilder(GEO_POINT_FIELD_NAME, "2,3");
         QueryShardContext context = createShardContext();
         QueryShardException e = expectThrows(QueryShardException.class,
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 48f43eefeb3..54f8536563d 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -32,6 +32,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.test.AbstractBuilderTestCase.STRING_ALIAS_FIELD_NAME;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -141,8 +142,6 @@ public class WildcardQueryBuilderTests extends AbstractQueryTestCase<WildcardQue
     }
 
     public void testIndexWildcard() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-
         QueryShardContext context = createShardContext();
         String index = context.getFullyQualifiedIndexName();
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index 40f6605edf1..cc224019100 100644
--- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -256,17 +256,6 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
         assertThat(query, either(instanceOf(FunctionScoreQuery.class)).or(instanceOf(FunctionScoreQuery.class)));
     }
 
-    /**
-     * Overridden here to ensure the test is only run if at least one type is
-     * present in the mappings. Functions require the field to be
-     * explicitly mapped
-     */
-    @Override
-    public void testToQuery() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testToQuery();
-    }
-
     public void testIllegalArguments() {
         expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder((QueryBuilder) null));
         expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder((ScoreFunctionBuilder<?>) null));
@@ -487,7 +476,6 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
     }
 
     public void testWeight1fStillProducesWeightFunction() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         String queryString = Strings.toString(jsonBuilder().startObject()
             .startObject("function_score")
             .startArray("functions")
@@ -650,12 +638,6 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
         assertEquals(json, 1, parsed.getMinScore(), 0.0001);
     }
 
-    @Override
-    public void testMustRewrite() throws IOException {
-        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        super.testMustRewrite();
-    }
-
     public void testRewrite() throws IOException {
         FunctionScoreQueryBuilder functionScoreQueryBuilder =
             new FunctionScoreQueryBuilder(new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java
index 1f83842eab2..8d67941639f 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java
@@ -31,11 +31,7 @@ import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.BucketOrder;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBoundsTests;
 import org.joda.time.DateTimeZone;
-import org.junit.Assume;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -141,7 +137,6 @@ public class DateHistogramTests extends BaseAggregationTestCase<DateHistogramAgg
     }
 
     public void testRewriteTimeZone() throws IOException {
-        Assume.assumeTrue(getCurrentTypes().length > 0); // we need mappings
         FormatDateTimeFormatter format = Joda.forPattern("strict_date_optional_time");
 
         try (Directory dir = newDirectory();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index a891f30b93d..fccec5f784f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -31,7 +31,6 @@ import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -40,7 +39,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.index.Index;
@@ -129,20 +127,16 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
     protected static Version indexVersionCreated;
 
     private static ServiceHolder serviceHolder;
+    private static ServiceHolder serviceHolderWithNoType;
     private static int queryNameId = 0;
     private static Settings nodeSettings;
     private static Index index;
-    private static String[] currentTypes;
-    protected static String[] randomTypes;
+    private static Index indexWithNoType;
 
     protected static Index getIndex() {
         return index;
     }
 
-    protected static String[] getCurrentTypes() {
-        return currentTypes;
-    }
-
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return Collections.emptyList();
     }
@@ -153,40 +147,12 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
     @BeforeClass
     public static void beforeClass() {
         nodeSettings = Settings.builder()
-                .put("node.name", AbstractQueryTestCase.class.toString())
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .build();
+            .put("node.name", AbstractQueryTestCase.class.toString())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .build();
 
-        index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
-
-        // Set a single type in the index
-        switch (random().nextInt(3)) {
-        case 0:
-            currentTypes = new String[0]; // no types
-            break;
-        default:
-            currentTypes = new String[] { "_doc" };
-            break;
-        }
-        randomTypes = getRandomTypes();
-    }
-
-    private static String[] getRandomTypes() {
-        String[] types;
-        if (currentTypes.length > 0 && randomBoolean()) {
-            int numberOfQueryTypes = randomIntBetween(1, currentTypes.length);
-            types = new String[numberOfQueryTypes];
-            for (int i = 0; i < numberOfQueryTypes; i++) {
-                types[i] = randomFrom(currentTypes);
-            }
-        } else {
-            if (randomBoolean()) {
-                types = new String[]{MetaData.ALL};
-            } else {
-                types = new String[0];
-            }
-        }
-        return types;
+        index = new Index(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLength(10));
+        indexWithNoType = new Index(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLength(10));
     }
 
     @Override
@@ -210,34 +176,37 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
     protected Settings indexSettings() {
         // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
         indexVersionCreated = randomBoolean() ? Version.CURRENT
-                : VersionUtils.randomVersionBetween(random(), null, Version.CURRENT);
+                : VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT);
         return Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated)
             .build();
     }
 
     protected static String expectedFieldName(String builderFieldName) {
-        if (currentTypes.length == 0) {
-            return builderFieldName;
-        }
         return ALIAS_TO_CONCRETE_FIELD_NAME.getOrDefault(builderFieldName, builderFieldName);
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
-        IOUtils.close(serviceHolder);
+        org.apache.lucene.util.IOUtils.close(serviceHolder);
+        org.apache.lucene.util.IOUtils.close(serviceHolderWithNoType);
         serviceHolder = null;
+        serviceHolderWithNoType = null;
     }
 
     @Before
     public void beforeTest() throws IOException {
         if (serviceHolder == null) {
-            serviceHolder = new ServiceHolder(nodeSettings, indexSettings(), getPlugins(), this);
+            serviceHolder = new ServiceHolder(nodeSettings, indexSettings(), getPlugins(), this, true);
         }
         serviceHolder.clientInvocationHandler.delegate = this;
+        if (serviceHolderWithNoType == null) {
+            serviceHolderWithNoType = new ServiceHolder(nodeSettings, indexSettings(), getPlugins(), this, false);
+        }
+        serviceHolderWithNoType.clientInvocationHandler.delegate = this;
     }
 
-    protected static SearchContext getSearchContext(String[] types, QueryShardContext context) {
+    protected static SearchContext getSearchContext(QueryShardContext context) {
         TestSearchContext testSearchContext = new TestSearchContext(context) {
             @Override
             public MapperService mapperService() {
@@ -250,13 +219,13 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
             }
 
         };
-        testSearchContext.getQueryShardContext().setTypes(types);
         return testSearchContext;
     }
 
     @After
     public void afterTest() {
         serviceHolder.clientInvocationHandler.delegate = null;
+        serviceHolderWithNoType.clientInvocationHandler.delegate = null;
     }
 
     /**
@@ -280,6 +249,13 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
         return serviceHolder.createShardContext(reader);
     }
 
+    /**
+     * @return a new {@link QueryShardContext} based on an index with no type registered
+     */
+    protected static QueryShardContext createShardContextWithNoType() {
+        return serviceHolderWithNoType.createShardContext(null);
+    }
+
     /**
      * @return a new {@link QueryShardContext} based on the base test index and queryParserService
      */
@@ -331,8 +307,11 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
         private final Client client;
         private final long nowInMillis = randomNonNegativeLong();
 
-        ServiceHolder(Settings nodeSettings, Settings indexSettings,
-                      Collection<Class<? extends Plugin>> plugins, AbstractBuilderTestCase testCase) throws IOException {
+        ServiceHolder(Settings nodeSettings,
+                        Settings indexSettings,
+                        Collection<Class<? extends Plugin>> plugins,
+                        AbstractBuilderTestCase testCase,
+                        boolean registerType) throws IOException {
             Environment env = InternalSettingsPreparer.prepareEnvironment(nodeSettings);
             PluginsService pluginsService;
             pluginsService = new PluginsService(nodeSettings, null, env.modulesFile(), env.pluginsFile(), plugins);
@@ -379,9 +358,8 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
                 }
             });
 
-
-            for (String type : currentTypes) {
-                mapperService.merge(type, new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef(type,
+            if (registerType) {
+                mapperService.merge("_doc", new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef("_doc",
                     STRING_FIELD_NAME, "type=text",
                     STRING_FIELD_NAME_2, "type=keyword",
                     STRING_ALIAS_FIELD_NAME, "type=alias,path=" + STRING_FIELD_NAME,
@@ -399,12 +377,12 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
                     GEO_SHAPE_FIELD_NAME, "type=geo_shape"
                 ))), MapperService.MergeReason.MAPPING_UPDATE);
                 // also add mappings for two inner field in the object field
-                mapperService.merge(type, new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\","
-                                + "\"properties\":{\"" + DATE_FIELD_NAME + "\":{\"type\":\"date\"},\"" +
-                                INT_FIELD_NAME + "\":{\"type\":\"integer\"}}}}}"),
-                        MapperService.MergeReason.MAPPING_UPDATE);
+                mapperService.merge("_doc", new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\","
+                        + "\"properties\":{\"" + DATE_FIELD_NAME + "\":{\"type\":\"date\"},\"" +
+                        INT_FIELD_NAME + "\":{\"type\":\"integer\"}}}}}"),
+                    MapperService.MergeReason.MAPPING_UPDATE);
+                testCase.initializeAdditionalMappings(mapperService);
             }
-            testCase.initializeAdditionalMappings(mapperService);
         }
 
         @Override
@@ -423,5 +401,4 @@ public abstract class AbstractBuilderTestCase extends ESTestCase {
             return new ScriptModule(Settings.EMPTY, scriptPlugins);
         }
     }
-
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index c1efd9d8e6a..f71edfde84f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -413,7 +413,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             context.setAllowUnmappedFields(true);
             QB firstQuery = createTestQueryBuilder();
             QB controlQuery = copyQuery(firstQuery);
-            SearchContext searchContext = getSearchContext(randomTypes, context);
+            SearchContext searchContext = getSearchContext(context);
             /* we use a private rewrite context here since we want the most realistic way of asserting that we are cacheable or not.
              * We do it this way in SearchService where
              * we first rewrite the query with a private context, then reset the context and then build the actual lucene query*/
@@ -443,7 +443,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 secondQuery.queryName(secondQuery.queryName() == null ? randomAlphaOfLengthBetween(1, 30) : secondQuery.queryName()
                         + randomAlphaOfLengthBetween(1, 10));
             }
-            searchContext = getSearchContext(randomTypes, context);
+            searchContext = getSearchContext(context);
             Query secondLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context);
             assertNotNull("toQuery should not return null", secondLuceneQuery);
             assertLuceneQuery(secondQuery, secondLuceneQuery, searchContext);
@@ -668,10 +668,11 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
      */
     protected static String getRandomFieldName() {
         // if no type is set then return a random field name
-        if (getCurrentTypes().length == 0 || randomBoolean()) {
+        if (randomBoolean()) {
             return randomAlphaOfLengthBetween(1, 10);
+        } else {
+            return randomFrom(MAPPED_LEAF_FIELD_NAMES);
         }
-        return randomFrom(MAPPED_LEAF_FIELD_NAMES);
     }
 
     /**

From 860f92fcdd46d5e91faf3c72d4fd7732b43f46b5 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Thu, 26 Jul 2018 20:29:27 +0200
Subject: [PATCH 205/260] Copy missing segment attributes in getSegmentInfo
 (#32396)

The index sort and the attributes map of a segment are not copied
on committed segments that are not loaded by the internal or external searcher.
---
 server/src/main/java/org/elasticsearch/index/engine/Engine.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index 53a7baa60f6..b7c938b469f 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -800,6 +800,8 @@ public abstract class Engine implements Closeable {
                     } catch (IOException e) {
                         logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e);
                     }
+                    segment.segmentSort = info.info.getIndexSort();
+                    segment.attributes = info.info.getAttributes();
                     segments.put(info.info.name, segment);
                 } else {
                     segment.committed = true;

From 67a884ec88bd18a7866a82efe7b164b3b48459d2 Mon Sep 17 00:00:00 2001
From: lcawl <lcawley@elastic.co>
Date: Thu, 26 Jul 2018 12:28:26 -0700
Subject: [PATCH 206/260] [DOCS] Fixes formatting of scope object in job
 resource

---
 .../docs/en/rest-api/ml/jobresource.asciidoc  | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
index 3c4e330722f..fb3424cab3f 100644
--- a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
+++ b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
@@ -434,17 +434,20 @@ A rule has the following properties:
   as usual. This action is suitable when certain values are expected to be
   consistently anomalous and they affect the model in a way that negatively
   impacts the rest of the results.
+  
 `scope`::
-  (object) An optional scope of series where the rule applies. By default the scope
-  includes all series. Scoping is allowed for any of the partition/by/over fields.
-  To add a scope for a field add the field name as a key in the scope object and
-  set its value to an object with properties:
-    `filter_id`::
-      (string) The id of the <<ml-filter-resource,filter>> to be used.
-    `filter_type`::
-      (string) Either `include` (the rule applies for values in the filter)
-      or `exclude` (the rule applies for values not in the filter). Defaults
-      to `include`.
+  (object) An optional scope of series where the rule applies. By default, the 
+  scope includes all series. Scoping is allowed for any of the fields that are 
+  also specified in `by_field_name`, `over_field_name`, or `partition_field_name`.
+  To add a scope for a field, add the field name as a key in the scope object and
+  set its value to an object with the following properties:  
+`filter_id`:::
+  (string) The id of the <<ml-filter-resource,filter>> to be used.
+
+`filter_type`:::
+  (string) Either `include` (the rule applies for values in the filter)
+  or `exclude` (the rule applies for values not in the filter). Defaults
+  to `include`.    
 
 `conditions`::
   (array) An optional array of numeric conditions when the rule applies.

From 1628c833c7da15748011e17ba03c2bc51e312428 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Thu, 26 Jul 2018 21:36:49 +0200
Subject: [PATCH 207/260] TESTS: Move netty leak detection to paranoid level
 (#32354)

---
 .../src/main/java/org/elasticsearch/test/ESTestCase.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 44fe621dfa4..ee4f2f3b222 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -201,7 +201,7 @@ public abstract class ESTestCase extends LuceneTestCase {
         System.setProperty("log4j2.disable.jmx", "true");
 
         // Enable Netty leak detection and monitor logger for logged leak errors
-        System.setProperty("io.netty.leakDetection.level", "advanced");
+        System.setProperty("io.netty.leakDetection.level", "paranoid");
         String leakLoggerName = "io.netty.util.ResourceLeakDetector";
         Logger leakLogger = LogManager.getLogger(leakLoggerName);
         Appender leakAppender = new AbstractAppender(leakLoggerName, null,

From 9198bcf78e4ff75ddefeceebb7b139c9485e2b4c Mon Sep 17 00:00:00 2001
From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com>
Date: Fri, 27 Jul 2018 12:01:52 +1000
Subject: [PATCH 208/260] [Kerberos] Avoid vagrant update on precommit (#32416)

This commit avoids dependency during compile on copy keytab to
be present in the generated sources so pre-commit does not
stall for updating vagrant box.

Closes#32387
---
 x-pack/qa/kerberos-tests/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle
index 2981c0e4ec6..d2818bfc127 100644
--- a/x-pack/qa/kerberos-tests/build.gradle
+++ b/x-pack/qa/kerberos-tests/build.gradle
@@ -121,7 +121,7 @@ integTestRunner {
 if (project.rootProject.vagrantSupported == false) {
     integTest.enabled = false
 } else {
-    project.sourceSets.test.output.dir(generatedResources, builtBy: copyKeytabToGeneratedResources)
+    project.sourceSets.test.output.dir(generatedResources)
     integTestCluster.dependsOn krb5AddPrincipals, krb5kdcFixture, copyKeytabToGeneratedResources
     integTest.finalizedBy project(':test:fixtures:krb5kdc-fixture').halt
 }

From 53ff06e621213cb007d0e03b8ec8d80431d9186a Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 27 Jul 2018 08:28:51 +0200
Subject: [PATCH 209/260] Upgrade to Lucene-7.5.0-snapshot-608f0277b0 (#32390)

The main highlight is the removal of the reclaim_deletes_weight in the TieredMergePolicy.
The es setting index.merge.policy.reclaim_deletes_weight is deprecated in this commit and the value is ignored. The new merge policy setting setDeletesPctAllowed should be added in a follow up.
---
 buildSrc/version.properties                          |  2 +-
 docs/Versions.asciidoc                               |  4 ++--
 .../licenses/lucene-expressions-7.4.0.jar.sha1       |  1 -
 ...ne-expressions-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 plugins/analysis-icu/build.gradle                    |  2 +-
 plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1    |  1 -
 plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1    |  1 +
 .../licenses/lucene-analyzers-icu-7.4.0.jar.sha1     |  1 -
 ...-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 .../lucene-analyzers-kuromoji-7.4.0.jar.sha1         |  1 -
 ...yzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 .../licenses/lucene-analyzers-nori-7.4.0.jar.sha1    |  1 -
 ...analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 .../lucene-analyzers-phonetic-7.4.0.jar.sha1         |  1 -
 ...yzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 .../licenses/lucene-analyzers-smartcn-7.4.0.jar.sha1 |  1 -
 ...lyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 .../licenses/lucene-analyzers-stempel-7.4.0.jar.sha1 |  1 -
 ...lyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 .../lucene-analyzers-morfologik-7.4.0.jar.sha1       |  1 -
 ...ers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 .../index/store/SmbMMapDirectoryTests.java           |  8 ++++++++
 .../index/store/SmbSimpleFSDirectoryTests.java       |  8 ++++++++
 .../licenses/lucene-analyzers-common-7.4.0.jar.sha1  |  1 -
 ...alyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 .../licenses/lucene-backward-codecs-7.4.0.jar.sha1   |  1 -
 ...ackward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-core-7.4.0.jar.sha1           |  1 -
 .../lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1   |  1 +
 server/licenses/lucene-grouping-7.4.0.jar.sha1       |  1 -
 ...ucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-highlighter-7.4.0.jar.sha1    |  1 -
 ...ne-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-join-7.4.0.jar.sha1           |  1 -
 .../lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1   |  1 +
 server/licenses/lucene-memory-7.4.0.jar.sha1         |  1 -
 .../lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-misc-7.4.0.jar.sha1           |  1 -
 .../lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1   |  1 +
 server/licenses/lucene-queries-7.4.0.jar.sha1        |  1 -
 ...lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-queryparser-7.4.0.jar.sha1    |  1 -
 ...ne-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-sandbox-7.4.0.jar.sha1        |  1 -
 ...lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-spatial-7.4.0.jar.sha1        |  1 -
 ...lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-spatial-extras-7.4.0.jar.sha1 |  1 -
 ...spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-spatial3d-7.4.0.jar.sha1      |  1 -
 ...cene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/licenses/lucene-suggest-7.4.0.jar.sha1        |  1 -
 ...lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1 |  1 +
 server/src/main/java/org/elasticsearch/Version.java  |  2 +-
 .../org/elasticsearch/index/EsTieredMergePolicy.java | 11 +----------
 .../java/org/elasticsearch/index/IndexSettings.java  |  1 -
 .../org/elasticsearch/index/MergePolicyConfig.java   |  7 +------
 .../index/EsTieredMergePolicyTests.java              | 12 ++----------
 .../index/MergePolicySettingsTests.java              |  5 -----
 .../sql-action/licenses/lucene-core-7.4.0.jar.sha1   |  1 -
 .../lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1   |  1 +
 61 files changed, 50 insertions(+), 62 deletions(-)
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-7.4.0.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0.jar.sha1
 create mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0.jar.sha1
 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-analyzers-common-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-backward-codecs-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-core-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-grouping-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-highlighter-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-join-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-memory-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-misc-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-queries-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-queryparser-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-sandbox-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial-extras-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-spatial3d-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 server/licenses/lucene-suggest-7.4.0.jar.sha1
 create mode 100644 server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1
 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-7.4.0.jar.sha1
 create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 17e5cb5ff01..023d5d5b8dc 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0
+lucene            = 7.5.0-snapshot-608f0277b0
 
 # optional dependencies
 spatial4j         = 0.7
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index 948a3387f03..6e127a6ccfc 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -1,7 +1,7 @@
 :version:               7.0.0-alpha1
 :major-version:         7.x
-:lucene_version:        7.4.0
-:lucene_version_path:   7_4_0
+:lucene_version:        7.5.0
+:lucene_version_path:   7_5_0
 :branch:                master
 :jdk:                   1.8.0_131
 :jdk_major:             8
diff --git a/modules/lang-expression/licenses/lucene-expressions-7.4.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.4.0.jar.sha1
deleted file mode 100644
index 2b14a61f264..00000000000
--- a/modules/lang-expression/licenses/lucene-expressions-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9f0a326f7ec1671ffb07f95b27f1a5812b7dc1c3
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..908f70131b3
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+bd7d8078a2d0ad11a24f54156cc015630c96858a
\ No newline at end of file
diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle
index 4d6f9310cdd..ad5a7b7c57b 100644
--- a/plugins/analysis-icu/build.gradle
+++ b/plugins/analysis-icu/build.gradle
@@ -30,7 +30,7 @@ forbiddenApis {
 
 dependencies {
   compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}"
-  compile 'com.ibm.icu:icu4j:61.1'
+  compile 'com.ibm.icu:icu4j:62.1'
 }
 
 dependencyLicenses {
diff --git a/plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1 b/plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1
deleted file mode 100644
index 7b7fcfe1c79..00000000000
--- a/plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-28d33b5e44e72edcc66a5da7a34a42147f38d987
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 b/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1
new file mode 100644
index 00000000000..c24c69cf4b9
--- /dev/null
+++ b/plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1
@@ -0,0 +1 @@
+7a4d00d5ec5febd252a6182e8b6e87a0a9821f81
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0.jar.sha1
deleted file mode 100644
index b5291b30c7d..00000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-394e811e9d9bf0b9fba837f7ceca9e8f3e39d1c2
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..5b6947a9c75
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+7a37816def72a748416c4ae8b0f6817e30efb99f
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0.jar.sha1
deleted file mode 100644
index 49f55bea5e6..00000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5cd56acfa16ba20e19b5d21d90b510eada841431
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..d39638c1884
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+ca7437178cdbf7b8bfe0d75c75e3c8eb93925724
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0.jar.sha1
deleted file mode 100644
index c4b61b763b4..00000000000
--- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-db7b56f4cf533ad9022d2312c5ee48331edccca3
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..21c25d2bb24
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+3f5dec44f380d6d58bc1c8aec51964fcb5390b60
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0.jar.sha1
deleted file mode 100644
index 779cac97612..00000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e8dba4d28a595eab2e8fb6095d1ac5f2d3872144
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..f58c597eadd
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+453bf1d60df0415439095624e0b3e42492ad4716
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0.jar.sha1
deleted file mode 100644
index cf5c49a2759..00000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1243c771ee824c46a3d66ae3e4256d919fc06fbe
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..8ccec8dbf37
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+70095a45257bca9f46629b5fb6cedf9eff5e2b07
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0.jar.sha1
deleted file mode 100644
index 830b9ccf9cb..00000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c783794b0d20d8dc1285edc7701f386b1f0e2fb8
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..ec9c33119f5
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+7199d6962d268b7877f7b5160e98e4ff21cce5c7
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0.jar.sha1
deleted file mode 100644
index a96e05f5e3b..00000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9438efa504a89afb6cb4c66448c257f865164d23
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..ba9148ef1b3
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+12aff508d39d206a1aead5013ecd11882062eb06
\ No newline at end of file
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java
index 3dad35adaa2..f4f6bed7f04 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java
+++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java
@@ -30,4 +30,12 @@ public class SmbMMapDirectoryTests extends EsBaseDirectoryTestCase {
     protected Directory getDirectory(Path file) throws IOException {
         return new SmbDirectoryWrapper(new MMapDirectory(file));
     }
+
+    @Override
+    public void testCreateOutputForExistingFile() throws IOException {
+        /**
+         * This test is disabled because {@link SmbDirectoryWrapper} opens existing file
+         * with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
+         */
+    }
 }
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java
index 659f6eff7ac..c20b3bc2863 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java
+++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java
@@ -30,4 +30,12 @@ public class SmbSimpleFSDirectoryTests extends EsBaseDirectoryTestCase {
     protected Directory getDirectory(Path file) throws IOException {
         return new SmbDirectoryWrapper(new SimpleFSDirectory(file));
     }
+
+    @Override
+    public void testCreateOutputForExistingFile() throws IOException {
+        /**
+         * This test is disabled because {@link SmbDirectoryWrapper} opens existing file
+         * with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
+         */
+    }
 }
diff --git a/server/licenses/lucene-analyzers-common-7.4.0.jar.sha1 b/server/licenses/lucene-analyzers-common-7.4.0.jar.sha1
deleted file mode 100644
index 928cc6dea04..00000000000
--- a/server/licenses/lucene-analyzers-common-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e1afb580df500626a1c695e0fc9a7e8a8f58bcac
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..8b2a098a3a2
--- /dev/null
+++ b/server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+d27958843ca118db2ffd2c242ae3761bd5a47328
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-7.4.0.jar.sha1 b/server/licenses/lucene-backward-codecs-7.4.0.jar.sha1
deleted file mode 100644
index a94663119e7..00000000000
--- a/server/licenses/lucene-backward-codecs-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a6ad941ef1fdad48673ed511631b7e48a9456bf7
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..d8496a0a86a
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+7ea220ba8e4accb8b04e280463042ad470e23bc0
\ No newline at end of file
diff --git a/server/licenses/lucene-core-7.4.0.jar.sha1 b/server/licenses/lucene-core-7.4.0.jar.sha1
deleted file mode 100644
index 80ba6c76aa3..00000000000
--- a/server/licenses/lucene-core-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-730d9ac80436c8cbc0b2a8a749259be536b97316
\ No newline at end of file
diff --git a/server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..d38fb392c35
--- /dev/null
+++ b/server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+471096d6e92338b208aa91f3a85feb2f9cfc4afd
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-7.4.0.jar.sha1 b/server/licenses/lucene-grouping-7.4.0.jar.sha1
deleted file mode 100644
index 5b781d26829..00000000000
--- a/server/licenses/lucene-grouping-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-56f99858a4421a517b52da36a222debcccab80c6
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..7f83082fa0c
--- /dev/null
+++ b/server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+f0af947c60d24f779c22f774e81ebd7dd91cc932
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-7.4.0.jar.sha1 b/server/licenses/lucene-highlighter-7.4.0.jar.sha1
deleted file mode 100644
index e1ebb95fe1b..00000000000
--- a/server/licenses/lucene-highlighter-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5266b45d7f049662817d739881765904621876d0
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..6b9f2cb724d
--- /dev/null
+++ b/server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+fbc83ac5a0139ed7e7faf6c95a2718f46f28c641
\ No newline at end of file
diff --git a/server/licenses/lucene-join-7.4.0.jar.sha1 b/server/licenses/lucene-join-7.4.0.jar.sha1
deleted file mode 100644
index ff81c33c3f8..00000000000
--- a/server/licenses/lucene-join-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c77154d18c4944ceb6ce0741060632f57d623fdc
\ No newline at end of file
diff --git a/server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..a085943140e
--- /dev/null
+++ b/server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+30adfe493982b0db059dc243e269eea38d850d46
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-7.4.0.jar.sha1 b/server/licenses/lucene-memory-7.4.0.jar.sha1
deleted file mode 100644
index 7c0117dff6b..00000000000
--- a/server/licenses/lucene-memory-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-186ff981feec1bdbf1a6236e786ec171b5fbe3e0
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..7acc70be151
--- /dev/null
+++ b/server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+656f304261d9aad05070fb68593beffafe9147e3
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-7.4.0.jar.sha1 b/server/licenses/lucene-misc-7.4.0.jar.sha1
deleted file mode 100644
index 5cdf6810fa5..00000000000
--- a/server/licenses/lucene-misc-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bf844bb6f6d84da19e8c79ce5fbb4cf6d00f2611
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..e46b138ba7b
--- /dev/null
+++ b/server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+8bf22ad81a7480c255b55bada401eb131bfdb4df
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-7.4.0.jar.sha1 b/server/licenses/lucene-queries-7.4.0.jar.sha1
deleted file mode 100644
index 19889037937..00000000000
--- a/server/licenses/lucene-queries-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-229a50e6d9d4db076f671c230d493000c6e2972c
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..a7114feef62
--- /dev/null
+++ b/server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+edb3de4d68a34c1e1ca08f79fe4d103b10e98ad1
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-7.4.0.jar.sha1 b/server/licenses/lucene-queryparser-7.4.0.jar.sha1
deleted file mode 100644
index afdc275afe2..00000000000
--- a/server/licenses/lucene-queryparser-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8e58add0d0c39df97d07c8e343041989bf4b3a3f
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..cf3011c9a45
--- /dev/null
+++ b/server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+7ece30d5f1e18d96f61644451c858c3d9960558f
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-7.4.0.jar.sha1 b/server/licenses/lucene-sandbox-7.4.0.jar.sha1
deleted file mode 100644
index 81ae3bddd07..00000000000
--- a/server/licenses/lucene-sandbox-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1692604fa06a945d1ee19939022ef1a912235db3
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..30513e58bf6
--- /dev/null
+++ b/server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+ad3bd0c2ed96556193c7215bef328e689d0b157f
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-7.4.0.jar.sha1 b/server/licenses/lucene-spatial-7.4.0.jar.sha1
deleted file mode 100644
index cc3f31340b9..00000000000
--- a/server/licenses/lucene-spatial-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-847d2f897961124e2fc7d5e55d8309635bb026bc
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..6146b055c13
--- /dev/null
+++ b/server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+8a6bd97e39ee5af60126adbe8c8375dc41b1ea8e
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-7.4.0.jar.sha1 b/server/licenses/lucene-spatial-extras-7.4.0.jar.sha1
deleted file mode 100644
index 3f05790e430..00000000000
--- a/server/licenses/lucene-spatial-extras-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-586892eefc0546643d7f5d7f83659c7db0d534ff
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..c812f044927
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+07e748d2d80000a7a213f3405b82b6e26b452948
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-7.4.0.jar.sha1 b/server/licenses/lucene-spatial3d-7.4.0.jar.sha1
deleted file mode 100644
index 8c767b16c53..00000000000
--- a/server/licenses/lucene-spatial3d-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-32cd2854f39ff453a5d128ce40e11eea4168abbf
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..b5ad83ac9fe
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+fd737bd5562f3943618ee7e73a0aaffb6319fdb2
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-7.4.0.jar.sha1 b/server/licenses/lucene-suggest-7.4.0.jar.sha1
deleted file mode 100644
index 59d59cf7941..00000000000
--- a/server/licenses/lucene-suggest-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0cdc1a512032f8b23dd4b1add0f5cd06325addc3
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1 b/server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..452b96420f8
--- /dev/null
+++ b/server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+ff3f260d1dc8c18bc67f3c33aa84a0ad290daac5
\ No newline at end of file
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index 1fefbad0d74..e2ade339618 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -182,7 +182,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
     public static final Version V_7_0_0_alpha1 =
-        new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
+        new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
     public static final Version CURRENT = V_7_0_0_alpha1;
 
     static {
diff --git a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java
index 27a83961903..68edf3a9b18 100644
--- a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java
+++ b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java
@@ -52,7 +52,7 @@ final class EsTieredMergePolicy extends FilterMergePolicy {
 
     @Override
     public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
-        return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);    
+        return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
     }
 
     public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
@@ -108,13 +108,4 @@ final class EsTieredMergePolicy extends FilterMergePolicy {
     public double getSegmentsPerTier() {
         return regularMergePolicy.getSegmentsPerTier();
     }
-
-    public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
-        regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-        forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-    }
-
-    public double getReclaimDeletesWeight() {
-        return regularMergePolicy.getReclaimDeletesWeight();
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
index 08cacee6ae0..486515e6755 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -416,7 +416,6 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, mergePolicyConfig::setMaxMergesAtOnceExplicit);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, mergePolicyConfig::setSegmentsPerTier);
-        scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, mergePolicyConfig::setReclaimDeletesWeight);
 
         scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
             mergeSchedulerConfig::setMaxThreadAndMergeCount);
diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
index b6c1e124a63..8a264cd3cb7 100644
--- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
+++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
@@ -150,7 +150,7 @@ public final class MergePolicyConfig {
             Property.Dynamic, Property.IndexScope);
     public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING =
         Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d,
-            Property.Dynamic, Property.IndexScope);
+            Property.Dynamic, Property.IndexScope, Property.Deprecated);
     public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; // don't convert to Setting<> and register... we only set this in tests and register via a plugin
 
 
@@ -176,17 +176,12 @@ public final class MergePolicyConfig {
         mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
         mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
         mergePolicy.setSegmentsPerTier(segmentsPerTier);
-        mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
         if (logger.isTraceEnabled()) {
             logger.trace("using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}]",
                 forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight);
         }
     }
 
-    void setReclaimDeletesWeight(Double reclaimDeletesWeight) {
-        mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-    }
-
     void setSegmentsPerTier(Double segmentsPerTier) {
         mergePolicy.setSegmentsPerTier(segmentsPerTier);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java
index fdee707d97a..30f73b887f7 100644
--- a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java
@@ -29,16 +29,14 @@ public class EsTieredMergePolicyTests extends ESTestCase {
         assertEquals(
                 new TieredMergePolicy().getMaxMergedSegmentMB(),
                 policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
-        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
-        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+        assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
     }
 
     public void testSetMaxMergedSegmentMB() {
         EsTieredMergePolicy policy = new EsTieredMergePolicy();
         policy.setMaxMergedSegmentMB(10 * 1024);
         assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
-        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
-        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+        assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
     }
 
     public void testSetForceMergeDeletesPctAllowed() {
@@ -71,10 +69,4 @@ public class EsTieredMergePolicyTests extends ESTestCase {
         policy.setSegmentsPerTier(42);
         assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
     }
-
-    public void testSetReclaimDeletesWeight() {
-        EsTieredMergePolicy policy = new EsTieredMergePolicy();
-        policy.setReclaimDeletesWeight(42);
-        assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
index d8b03ffeedb..a8370095564 100644
--- a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
@@ -96,10 +96,6 @@ public class MergePolicySettingsTests extends ESTestCase {
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
 
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
-        indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
-
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
@@ -110,7 +106,6 @@ public class MergePolicySettingsTests extends ESTestCase {
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
     }
 
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.4.0.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.4.0.jar.sha1
deleted file mode 100644
index 80ba6c76aa3..00000000000
--- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.4.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-730d9ac80436c8cbc0b2a8a749259be536b97316
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
new file mode 100644
index 00000000000..d38fb392c35
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1
@@ -0,0 +1 @@
+471096d6e92338b208aa91f3a85feb2f9cfc4afd
\ No newline at end of file

From 5decb23687eb19ed1d229c19382897f5010fb5ed Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jimczi@apache.org>
Date: Fri, 27 Jul 2018 10:50:22 +0200
Subject: [PATCH 210/260] bump lucene version after backport

---
 server/src/main/java/org/elasticsearch/Version.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index e2ade339618..a815a9711d0 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -179,7 +179,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final int V_6_4_0_ID = 6040099;
     public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_6_5_0_ID = 6050099;
-    public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
+    public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
     public static final Version V_7_0_0_alpha1 =
         new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);

From 83d007e7bee9763ab4b15553110d1b7a829f614b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 27 Jul 2018 11:50:13 +0200
Subject: [PATCH 211/260] [TEST] Mute failing testConvertLongHexError

See #32370
---
 .../ingest/common/ConvertProcessorTests.java        | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
index ae27c7850da..500f1aa1719 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
@@ -19,11 +19,6 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.test.ESTestCase;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -31,12 +26,17 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.test.ESTestCase;
+
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.elasticsearch.ingest.common.ConvertProcessor.Type;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.sameInstance;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class ConvertProcessorTests extends ESTestCase {
 
@@ -138,6 +138,7 @@ public class ConvertProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L));
     }
 
+    @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
     public void testConvertLongHexError() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String value = "0x" + randomAlphaOfLengthBetween(1, 10);

From 7aa53654976ce9c9932d36dca8ed81a342cce70b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 27 Jul 2018 14:41:32 +0200
Subject: [PATCH 212/260] [TEST] Mute failing
 InternalEngineTests#testSeqNoAndCheckpoints

---
 .../index/engine/InternalEngineTests.java     | 68 +++++++++----------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 87b63dfdef8..f47cd11e1b4 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -19,9 +19,41 @@
 
 package org.elasticsearch.index.engine;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
+import java.util.function.ToLongBiFunction;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
-
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -123,39 +155,6 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.LongSupplier;
-import java.util.function.Supplier;
-import java.util.function.ToLongBiFunction;
-import java.util.stream.Collectors;
-import java.util.stream.LongStream;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.shuffle;
 import static org.elasticsearch.index.engine.Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY;
@@ -2025,6 +2024,7 @@ public class InternalEngineTests extends EngineTestCase {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32430")
     public void testSeqNoAndCheckpoints() throws IOException {
         final int opCount = randomIntBetween(1, 256);
         long primarySeqNo = SequenceNumbers.NO_OPS_PERFORMED;

From 3ac57f0ba37aa2cdf24cc85d8037ffc22816cf6a Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 27 Jul 2018 10:40:16 -0400
Subject: [PATCH 213/260] Fix ordering of bootstrap checks in docs (#32417)

In the section of the bootstrap checks docs for the maximum map count
check, we refer to max size virtual memory check and explicitly call out
the maximum size virtual memory check as being the previous
point. However, this is not correct as the previous point is currently
the max file size check. It does make sense for these two checks to be
proximate to each other in the docs so this commit reorders the checks
so that the maximum size virtual memory check indeed comes before the
maximum map count check. This makes the sense in the maximum map count
check correct.
---
 .../reference/setup/bootstrap-checks.asciidoc | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index 4670bb87580..a8b8dd82d61 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -118,6 +118,19 @@ least 4096 threads. This can be done via `/etc/security/limits.conf`
 using the `nproc` setting (note that you might have to increase the
 limits for the `root` user too).
 
+=== Max file size check
+
+The segment files that are the components of individual shards and the translog
+generations that are components of the translog can get large (exceeding
+multiple gigabytes). On systems where the max size of files that can be created
+by the Elasticsearch process is limited, this can lead to failed
+writes. Therefore, the safest option here is that the max file size is unlimited
+and that is what the max file size bootstrap check enforces. To pass the max
+file check, you must configure your system to allow the Elasticsearch process
+the ability to write files of unlimited size. This can be done via
+`/etc/security/limits.conf` using the `fsize` setting to `unlimited` (note that
+you might have to increase the limits for the `root` user too).
+
 [[max-size-virtual-memory-check]]
 === Maximum size virtual memory check
 
@@ -133,19 +146,6 @@ address space. This can be done via `/etc/security/limits.conf` using
 the `as` setting to `unlimited` (note that you might have to increase
 the limits for the `root` user too).
 
-=== Max file size check
-
-The segment files that are the components of individual shards and the translog
-generations that are components of the translog can get large (exceeding
-multiple gigabytes). On systems where the max size of files that can be created
-by the Elasticsearch process is limited, this can lead to failed
-writes. Therefore, the safest option here is that the max file size is unlimited
-and that is what the max file size bootstrap check enforces. To pass the max
-file check, you must configure your system to allow the Elasticsearch process
-the ability to write files of unlimited size. This can be done via
-`/etc/security/limits.conf` using the `fsize` setting to `unlimited` (note that
-you might have to increase the limits for the `root` user too).
-
 === Maximum map count check
 
 Continuing from the previous <<max-size-virtual-memory-check,point>>, to

From dcb5d24639ce073d354702c8fe1f038ce9f948bb Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 27 Jul 2018 17:08:54 +0200
Subject: [PATCH 214/260] [TEST] Mute failing FlushIT test

See #32436
---
 .../elasticsearch/indices/flush/FlushIT.java  | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
index 185a9b5cdce..bdd01e88cca 100644
--- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
@@ -18,6 +18,16 @@
  */
 package org.elasticsearch.indices.flush;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
 import org.apache.lucene.index.Term;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
@@ -47,16 +57,6 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
@@ -286,6 +286,7 @@ public class FlushIT extends ESIntegTestCase {
         assertThat(fullResult.successfulShards(), equalTo(numberOfReplicas + 1));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32436")
     public void testDoNotRenewSyncedFlushWhenAllSealed() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(between(2, 3));
         final int numberOfReplicas = internalCluster().numDataNodes() - 1;

From 588db621ac089cdb193f0500078e736e7d36d6ce Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 27 Jul 2018 11:24:03 -0400
Subject: [PATCH 215/260] Remove reference to non-existent store type (#32418)

We removed the default_fs store type yet the docs still contain a
reference to them. This commit addresses that by removing this
reference, and changing a reference to this section of the docs to
instead refer to mmapfs.
---
 docs/reference/index-modules/store.asciidoc            | 5 -----
 docs/reference/setup/sysconfig/virtual-memory.asciidoc | 2 +-
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc
index e657affb607..a1e00bac616 100644
--- a/docs/reference/index-modules/store.asciidoc
+++ b/docs/reference/index-modules/store.asciidoc
@@ -67,11 +67,6 @@ process equal to the size of the file being mapped. Before using this
 class, be sure you have allowed plenty of
 <<vm-max-map-count,virtual address space>>.
 
-[[default_fs]]`default_fs` deprecated[5.0.0, The `default_fs` store type is deprecated - use `fs` instead]::
-
-The `default` type is deprecated and is aliased to `fs` for backward
-compatibility.
-
 === Pre-loading data into the file system cache
 
 NOTE: This is an expert setting, the details of which may change in the future.
diff --git a/docs/reference/setup/sysconfig/virtual-memory.asciidoc b/docs/reference/setup/sysconfig/virtual-memory.asciidoc
index 672a14ef50d..67aff16d24a 100644
--- a/docs/reference/setup/sysconfig/virtual-memory.asciidoc
+++ b/docs/reference/setup/sysconfig/virtual-memory.asciidoc
@@ -1,7 +1,7 @@
 [[vm-max-map-count]]
 === Virtual memory
 
-Elasticsearch uses a <<default_fs,`mmapfs`>> directory by
+Elasticsearch uses a <<mmapfs,`mmapfs`>> directory by
 default to store its indices.  The default operating system limits on mmap
 counts is likely to be too low, which may result in out of memory exceptions.
 

From e9e1095596ace4cd50c5a09a6860506add1eb216 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 27 Jul 2018 14:23:37 -0700
Subject: [PATCH 216/260] Painless: Add method type to method. (#32441)

MethodType can be computed at compile-time rather than run-time. This removes the
method that collects MethodType at run-time from a PainlessMethod since is it no longer
necessary.
---
 .../elasticsearch/painless/FunctionRef.java   |  6 +--
 .../lookup/PainlessLookupBuilder.java         | 13 +++--
 .../painless/lookup/PainlessMethod.java       | 54 ++-----------------
 .../painless/node/SFunction.java              |  4 +-
 4 files changed, 20 insertions(+), 57 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index aa72724b930..7b7e8d25f39 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -93,12 +93,12 @@ public class FunctionRef {
      * @param numCaptures number of captured arguments
      */
     public FunctionRef(Class<?> expected, PainlessMethod interfaceMethod, PainlessMethod delegateMethod, int numCaptures) {
-        MethodType delegateMethodType = delegateMethod.getMethodType();
+        MethodType delegateMethodType = delegateMethod.methodType;
 
         interfaceMethodName = interfaceMethod.name;
         factoryMethodType = MethodType.methodType(expected,
                 delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
-        interfaceMethodType = interfaceMethod.getMethodType().dropParameterTypes(0, 1);
+        interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
 
         // the Painless$Script class can be inferred if owner is null
         if (delegateMethod.target == null) {
@@ -142,7 +142,7 @@ public class FunctionRef {
         interfaceMethodName = interfaceMethod.name;
         factoryMethodType = MethodType.methodType(expected,
             delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
-        interfaceMethodType = interfaceMethod.getMethodType().dropParameterTypes(0, 1);
+        interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
 
         delegateClassName = CLASS_NAME;
         delegateInvokeType = H_INVOKESTATIC;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 3675cc7cd0f..ba48fbea734 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -27,6 +27,7 @@ import org.elasticsearch.painless.spi.WhitelistMethod;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
@@ -356,10 +357,12 @@ public class PainlessLookupBuilder {
                         "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
             }
 
+            MethodType methodType = methodHandle.type();
+
             painlessConstructor = painlessMethodCache.computeIfAbsent(
                     new PainlessMethodCacheKey(targetClass, CONSTRUCTOR_NAME, typeParameters),
                     key -> new PainlessMethod(CONSTRUCTOR_NAME, targetClass, null, void.class, typeParameters,
-                                              asmConstructor, javaConstructor.getModifiers(), methodHandle)
+                                              asmConstructor, javaConstructor.getModifiers(), methodHandle, methodType)
             );
 
             painlessClassBuilder.constructors.put(painlessMethodKey, painlessConstructor);
@@ -516,10 +519,12 @@ public class PainlessLookupBuilder {
                             "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae);
                 }
 
+                MethodType methodType = methodHandle.type();
+
                 painlessMethod = painlessMethodCache.computeIfAbsent(
                         new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
                         key -> new PainlessMethod(methodName, targetClass, null, returnType,
-                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle));
+                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle, methodType));
 
                 painlessClassBuilder.staticMethods.put(painlessMethodKey, painlessMethod);
             } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
@@ -557,10 +562,12 @@ public class PainlessLookupBuilder {
                     }
                 }
 
+                MethodType methodType = methodHandle.type();
+
                 painlessMethod = painlessMethodCache.computeIfAbsent(
                         new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
                         key -> new PainlessMethod(methodName, targetClass, augmentedClass, returnType,
-                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle));
+                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle, methodType));
 
                 painlessClassBuilder.methods.put(painlessMethodKey, painlessMethod);
             } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
index 3321de94a26..904cf06907f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
@@ -38,9 +38,10 @@ public class PainlessMethod {
     public final org.objectweb.asm.commons.Method method;
     public final int modifiers;
     public final MethodHandle handle;
+    public final MethodType methodType;
 
     public PainlessMethod(String name, Class<?> target, Class<?> augmentation, Class<?> rtn, List<Class<?>> arguments,
-                          org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) {
+                          org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle, MethodType methodType) {
         this.name = name;
         this.augmentation = augmentation;
         this.target = target;
@@ -49,54 +50,7 @@ public class PainlessMethod {
         this.method = method;
         this.modifiers = modifiers;
         this.handle = handle;
-    }
-
-    /**
-     * Returns MethodType for this method.
-     * <p>
-     * This works even for user-defined Methods (where the MethodHandle is null).
-     */
-    public MethodType getMethodType() {
-        // we have a methodhandle already (e.g. whitelisted class)
-        // just return its type
-        if (handle != null) {
-            return handle.type();
-        }
-        // otherwise compute it
-        final Class<?> params[];
-        final Class<?> returnValue;
-        if (augmentation != null) {
-            // static method disguised as virtual/interface method
-            params = new Class<?>[1 + arguments.size()];
-            params[0] = augmentation;
-            for (int i = 0; i < arguments.size(); i++) {
-                params[i + 1] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
-            }
-            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
-        } else if (Modifier.isStatic(modifiers)) {
-            // static method: straightforward copy
-            params = new Class<?>[arguments.size()];
-            for (int i = 0; i < arguments.size(); i++) {
-                params[i] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
-            }
-            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
-        } else if ("<init>".equals(name)) {
-            // constructor: returns the owner class
-            params = new Class<?>[arguments.size()];
-            for (int i = 0; i < arguments.size(); i++) {
-                params[i] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
-            }
-            returnValue = target;
-        } else {
-            // virtual/interface method: add receiver class
-            params = new Class<?>[1 + arguments.size()];
-            params[0] = target;
-            for (int i = 0; i < arguments.size(); i++) {
-                params[i + 1] = PainlessLookupUtility.typeToJavaType(arguments.get(i));
-            }
-            returnValue = PainlessLookupUtility.typeToJavaType(rtn);
-        }
-        return MethodType.methodType(returnValue, params);
+        this.methodType = methodType;
     }
 
     public void write(MethodWriter writer) {
@@ -118,7 +72,7 @@ public class PainlessMethod {
             // method since java 8 did not check, but java 9 and 10 do
             if (Modifier.isInterface(clazz.getModifiers())) {
                 writer.visitMethodInsn(Opcodes.INVOKESTATIC,
-                        type.getInternalName(), name, getMethodType().toMethodDescriptorString(), true);
+                        type.getInternalName(), name, methodType.toMethodDescriptorString(), true);
             } else {
                 writer.invokeStatic(type, method);
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 7c243e296c7..600ca95504a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -145,9 +145,11 @@ public final class SFunction extends AStatement {
             }
         }
 
+        int modifiers = Modifier.STATIC | Modifier.PRIVATE;
         org.objectweb.asm.commons.Method method = new org.objectweb.asm.commons.Method(name, MethodType.methodType(
                 PainlessLookupUtility.typeToJavaType(rtnType), paramClasses).toMethodDescriptorString());
-        this.method = new PainlessMethod(name, null, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null);
+        MethodType methodType = MethodType.methodType(PainlessLookupUtility.typeToJavaType(rtnType), paramClasses);
+        this.method = new PainlessMethod(name, null, null, rtnType, paramTypes, method, modifiers, null, methodType);
     }
 
     @Override

From 139631c77d1a6bac002f82952d599a386122b1e3 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Fri, 27 Jul 2018 20:48:17 -0400
Subject: [PATCH 217/260] AwaitsFix IndexShardTests#testDocStats

Relates #32449
---
 .../test/java/org/elasticsearch/index/shard/IndexShardTests.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 1880b6b0954..d7b1be847c0 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -2376,6 +2376,7 @@ public class IndexShardTests extends IndexShardTestCase {
         closeShards(sourceShard, targetShard);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32449")
     public void testDocStats() throws IOException {
         IndexShard indexShard = null;
         try {

From 6e98615cc1b324f9c2f543125c4cc7a3d01f3591 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Fri, 27 Jul 2018 23:14:24 -0400
Subject: [PATCH 218/260] TEST: Avoid deletion in FlushIT

Due to the recent change in LUCENE-8263, a merge can be triggered if the
deletion ration is higher than 33%. An in-progress merge can prevent a
synced-flush from issuing.

This commit avoids deletes by using different docIds.

Closes #32436
---
 .../src/test/java/org/elasticsearch/indices/flush/FlushIT.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
index bdd01e88cca..4483ce0d606 100644
--- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
@@ -286,7 +286,6 @@ public class FlushIT extends ESIntegTestCase {
         assertThat(fullResult.successfulShards(), equalTo(numberOfReplicas + 1));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32436")
     public void testDoNotRenewSyncedFlushWhenAllSealed() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(between(2, 3));
         final int numberOfReplicas = internalCluster().numDataNodes() - 1;
@@ -311,7 +310,7 @@ public class FlushIT extends ESIntegTestCase {
         // Shards were updated, renew synced flush.
         final int moreDocs = between(1, 10);
         for (int i = 0; i < moreDocs; i++) {
-            index("test", "doc", Integer.toString(i));
+            index("test", "doc", "more-" + i);
         }
         final ShardsSyncedFlushResult thirdSeal = SyncedFlushUtil.attemptSyncedFlush(logger, internalCluster(), shardId);
         assertThat(thirdSeal.successfulShards(), equalTo(numberOfReplicas + 1));

From 5b1ad8099bfd0738bb16331e6f6952b8726b849e Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Sat, 28 Jul 2018 07:41:30 -0400
Subject: [PATCH 219/260] TEST: testDocStats should always use forceMerge
 (#32450)

Due to the recent change in LUCENE-8263, we need to adjust the deletion
ration to between 10% to 33% to preserve the current behavior of the
test. However, we may need another refinement if soft-deletes is enabled
as the actual deletes are different because of delete tombstones.

This commit prefers to always execute forceMerge instead of adjusting
the deletion ratio so that this test can focus on testing docStats.

Closes #32449
---
 .../java/org/elasticsearch/index/shard/IndexShardTests.java | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index d7b1be847c0..6d9e15c52af 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -2376,15 +2376,12 @@ public class IndexShardTests extends IndexShardTestCase {
         closeShards(sourceShard, targetShard);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32449")
     public void testDocStats() throws IOException {
         IndexShard indexShard = null;
         try {
             indexShard = newStartedShard();
             final long numDocs = randomIntBetween(2, 32); // at least two documents so we have docs to delete
-            // Delete at least numDocs/10 documents otherwise the number of deleted docs will be below 10%
-            // and forceMerge will refuse to expunge deletes
-            final long numDocsToDelete = randomIntBetween((int) Math.ceil(Math.nextUp(numDocs / 10.0)), Math.toIntExact(numDocs));
+            final long numDocsToDelete = randomLongBetween(1, numDocs);
             for (int i = 0; i < numDocs; i++) {
                 final String id = Integer.toString(i);
                 indexDoc(indexShard, "_doc", id);
@@ -2434,7 +2431,6 @@ public class IndexShardTests extends IndexShardTestCase {
 
             // merge them away
             final ForceMergeRequest forceMergeRequest = new ForceMergeRequest();
-            forceMergeRequest.onlyExpungeDeletes(randomBoolean());
             forceMergeRequest.maxNumSegments(1);
             indexShard.forceMerge(forceMergeRequest);
 

From 14a4a740ace4c96cf02f79af145c1ee7ec2401e8 Mon Sep 17 00:00:00 2001
From: David Roberts <dave.roberts@elastic.co>
Date: Mon, 30 Jul 2018 09:04:17 +0100
Subject: [PATCH 220/260] [CI] Mute DocumentSubsetReaderTests testSearch

Relates #32457
---
 .../security/authz/accesscontrol/DocumentSubsetReaderTests.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
index 0fb3094fca5..38857e2170d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
@@ -80,6 +80,7 @@ public class DocumentSubsetReaderTests extends ESTestCase {
         bitsetFilterCache.close();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32457")
     public void testSearch() throws Exception {
         IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig());
 

From 9a4d0069f6e179cb0db0578c3bfcdf16f6e0e6a2 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Mon, 30 Jul 2018 13:43:40 +0200
Subject: [PATCH 221/260] REST high-level client: parse back _ignored meta
 field (#32362)

`GetResult` and `SearchHit` have been adjusted to parse back the `_ignored` meta field whenever it gets printed out. Expanded the existing tests to make sure this is covered. Fixed also a small problem around highlighted fields in `SearchHitTests`.
---
 .../elasticsearch/index/get/GetResult.java    | 24 +++++--
 .../org/elasticsearch/search/SearchHit.java   | 33 +++++++---
 .../index/get/DocumentFieldTests.java         | 34 +++++++---
 .../index/get/GetResultTests.java             | 11 ++--
 .../elasticsearch/search/SearchHitTests.java  | 66 ++++++++-----------
 .../search/SearchSortValuesTests.java         | 14 ++--
 .../org/elasticsearch/test/RandomObjects.java | 20 +++---
 7 files changed, 116 insertions(+), 86 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index a3f83609037..ba5c4cd929f 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.get;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.document.DocumentField;
@@ -30,6 +31,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.IgnoredFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.search.lookup.SourceLookup;
 
@@ -225,10 +227,13 @@ public class GetResult implements Streamable, Iterable<DocumentField>, ToXConten
                 }
             }
         }
-
         for (DocumentField field : metaFields) {
-            Object value = field.getValue();
-            builder.field(field.getName(), value);
+            // TODO: can we avoid having an exception here?
+            if (field.getName().equals(IgnoredFieldMapper.NAME)) {
+                builder.field(field.getName(), field.getValues());
+            } else {
+                builder.field(field.getName(), field.<Object>getValue());
+            }
         }
 
         builder.field(FOUND, exists);
@@ -316,7 +321,11 @@ public class GetResult implements Streamable, Iterable<DocumentField>, ToXConten
                     parser.skipChildren(); // skip potential inner objects for forward compatibility
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                parser.skipChildren(); // skip potential inner arrays for forward compatibility
+                if (IgnoredFieldMapper.NAME.equals(currentFieldName)) {
+                    fields.put(currentFieldName, new DocumentField(currentFieldName, parser.list()));
+                } else {
+                    parser.skipChildren(); // skip potential inner arrays for forward compatibility
+                }
             }
         }
         return new GetResult(index, type, id, version, found, source, fields);
@@ -400,7 +409,12 @@ public class GetResult implements Streamable, Iterable<DocumentField>, ToXConten
 
     @Override
     public int hashCode() {
-        return Objects.hash(index, type, id, version, exists, fields, sourceAsMap());
+        return Objects.hash(version, exists, index, type, id, fields, sourceAsMap());
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
     }
 }
 
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java
index 8c688cbf446..66999c7e389 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -602,16 +602,24 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable<D
         for (String metadatafield : MapperService.getAllMetaFields()) {
             if (metadatafield.equals(Fields._ID) == false && metadatafield.equals(Fields._INDEX) == false
                     && metadatafield.equals(Fields._TYPE) == false) {
-                parser.declareField((map, field) -> {
-                    @SuppressWarnings("unchecked")
-                    Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
-                            v -> new HashMap<String, DocumentField>());
-                    fieldMap.put(field.getName(), field);
-                }, (p, c) -> {
-                    List<Object> values = new ArrayList<>();
-                    values.add(parseFieldsValue(p));
-                    return new DocumentField(metadatafield, values);
-                }, new ParseField(metadatafield), ValueType.VALUE);
+                if (metadatafield.equals(IgnoredFieldMapper.NAME)) {
+                    parser.declareObjectArray((map, list) -> {
+                            @SuppressWarnings("unchecked")
+                            Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
+                                v -> new HashMap<String, DocumentField>());
+                            DocumentField field = new DocumentField(metadatafield, list);
+                            fieldMap.put(field.getName(), field);
+                        }, (p, c) -> parseFieldsValue(p),
+                        new ParseField(metadatafield));
+                } else {
+                    parser.declareField((map, field) -> {
+                            @SuppressWarnings("unchecked")
+                            Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
+                                v -> new HashMap<String, DocumentField>());
+                            fieldMap.put(field.getName(), field);
+                        }, (p, c) -> new DocumentField(metadatafield, Collections.singletonList(parseFieldsValue(p))),
+                        new ParseField(metadatafield), ValueType.VALUE);
+                }
             }
         }
     }
@@ -958,4 +966,9 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable<D
             return Objects.hash(field, offset, child);
         }
     }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java b/server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java
index 830e41996e9..e2e3d4df67c 100644
--- a/server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java
+++ b/server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java
@@ -26,7 +26,11 @@ import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.mapper.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.IndexFieldMapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.RandomObjects;
 
@@ -98,14 +102,28 @@ public class DocumentFieldTests extends ESTestCase {
 
     public static Tuple<DocumentField, DocumentField> randomDocumentField(XContentType xContentType) {
         if (randomBoolean()) {
-            String fieldName = randomFrom(RoutingFieldMapper.NAME);
-            DocumentField documentField = new DocumentField(fieldName, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
+            String metaField = randomValueOtherThanMany(field -> field.equals(TypeFieldMapper.NAME)
+                    || field.equals(IndexFieldMapper.NAME) || field.equals(IdFieldMapper.NAME),
+                () -> randomFrom(MapperService.getAllMetaFields()));
+            DocumentField documentField;
+            if (metaField.equals(IgnoredFieldMapper.NAME)) {
+                int numValues = randomIntBetween(1, 3);
+                List<Object> ignoredFields = new ArrayList<>(numValues);
+                for (int i = 0; i < numValues; i++) {
+                    ignoredFields.add(randomAlphaOfLengthBetween(3, 10));
+                }
+                documentField = new DocumentField(metaField, ignoredFields);
+            } else {
+                //meta fields are single value only, besides _ignored
+                documentField = new DocumentField(metaField, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
+            }
             return Tuple.tuple(documentField, documentField);
+        } else {
+            String fieldName = randomAlphaOfLengthBetween(3, 10);
+            Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
+            DocumentField input = new DocumentField(fieldName, tuple.v1());
+            DocumentField expected = new DocumentField(fieldName, tuple.v2());
+            return Tuple.tuple(input, expected);
         }
-        String fieldName = randomAlphaOfLengthBetween(3, 10);
-        Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
-        DocumentField input = new DocumentField(fieldName, tuple.v1());
-        DocumentField expected = new DocumentField(fieldName, tuple.v2());
-        return Tuple.tuple(input, expected);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java b/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
index a38d183299c..1cc2612041f 100644
--- a/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
+++ b/server/src/test/java/org/elasticsearch/index/get/GetResultTests.java
@@ -90,7 +90,6 @@ public class GetResultTests extends ESTestCase {
         XContentType xContentType = randomFrom(XContentType.values());
         Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
         GetResult getResult = tuple.v1();
-
         // We don't expect to retrieve the index/type/id of the GetResult because they are not rendered
         // by the toXContentEmbedded method.
         GetResult expectedGetResult = new GetResult(null, null, null, -1,
@@ -106,7 +105,6 @@ public class GetResultTests extends ESTestCase {
             parsedEmbeddedGetResult = GetResult.fromXContentEmbedded(parser);
             assertNull(parser.nextToken());
         }
-
         assertEquals(expectedGetResult, parsedEmbeddedGetResult);
         //print the parsed object out and test that the output is the same as the original output
         BytesReference finalBytes = toXContentEmbedded(parsedEmbeddedGetResult, xContentType, humanReadable);
@@ -203,16 +201,17 @@ public class GetResultTests extends ESTestCase {
         return Tuple.tuple(getResult, expectedGetResult);
     }
 
-    private static Tuple<Map<String, DocumentField>,Map<String, DocumentField>> randomDocumentFields(XContentType xContentType) {
+    public static Tuple<Map<String, DocumentField>,Map<String, DocumentField>> randomDocumentFields(XContentType xContentType) {
         int numFields = randomIntBetween(2, 10);
         Map<String, DocumentField> fields = new HashMap<>(numFields);
         Map<String, DocumentField> expectedFields = new HashMap<>(numFields);
-        for (int i = 0; i < numFields; i++) {
+        while (fields.size() < numFields) {
             Tuple<DocumentField, DocumentField> tuple = randomDocumentField(xContentType);
             DocumentField getField = tuple.v1();
             DocumentField expectedGetField = tuple.v2();
-            fields.put(getField.getName(), getField);
-            expectedFields.put(expectedGetField.getName(), expectedGetField);
+            if (fields.putIfAbsent(getField.getName(), getField) == null) {
+                assertNull(expectedFields.putIfAbsent(expectedGetField.getName(), expectedGetField));
+            }
         }
         return Tuple.tuple(fields, expectedFields);
     }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
index 97dfad46454..87b8ba2dc59 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
@@ -19,30 +19,6 @@
 
 package org.elasticsearch.search;
 
-import org.apache.lucene.search.Explanation;
-import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.document.DocumentField;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.InputStreamStreamInput;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.SearchHit.NestedIdentity;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightFieldTests;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.RandomObjects;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -50,9 +26,31 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.function.Predicate;
 
+import org.apache.lucene.search.Explanation;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.get.GetResultTests;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchHit.NestedIdentity;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightFieldTests;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
@@ -63,8 +61,6 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class SearchHitTests extends ESTestCase {
 
-    private static Set<String> META_FIELDS = Sets.newHashSet("_uid", "_parent", "_routing", "_size", "_timestamp", "_ttl");
-
     public static SearchHit createTestItem(boolean withOptionalInnerHits) {
         int internalId = randomInt();
         String uid = randomAlphaOfLength(10);
@@ -75,18 +71,7 @@ public class SearchHitTests extends ESTestCase {
         }
         Map<String, DocumentField> fields = new HashMap<>();
         if (randomBoolean()) {
-            int size = randomIntBetween(0, 10);
-            for (int i = 0; i < size; i++) {
-                Tuple<List<Object>, List<Object>> values = RandomObjects.randomStoredFieldValues(random(),
-                        XContentType.JSON);
-                if (randomBoolean()) {
-                    String metaField = randomFrom(META_FIELDS);
-                    fields.put(metaField, new DocumentField(metaField, values.v1()));
-                } else {
-                    String fieldName = randomAlphaOfLengthBetween(5, 10);
-                    fields.put(fieldName, new DocumentField(fieldName, values.v1()));
-                }
-            }
+            fields = GetResultTests.randomDocumentFields(XContentType.JSON).v1();
         }
         SearchHit hit = new SearchHit(internalId, uid, type, nestedIdentity, fields);
         if (frequently()) {
@@ -109,7 +94,8 @@ public class SearchHitTests extends ESTestCase {
             int size = randomIntBetween(0, 5);
             Map<String, HighlightField> highlightFields = new HashMap<>(size);
             for (int i = 0; i < size; i++) {
-                highlightFields.put(randomAlphaOfLength(5), HighlightFieldTests.createTestItem());
+                HighlightField testItem = HighlightFieldTests.createTestItem();
+                highlightFields.put(testItem.getName(), testItem);
             }
             hit.highlightFields(highlightFields);
         }
diff --git a/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java b/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java
index d1a9a15a393..d69039b72f5 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchSortValuesTests.java
@@ -46,13 +46,13 @@ public class SearchSortValuesTests extends ESTestCase {
         List<Supplier<Object>> valueSuppliers = new ArrayList<>();
         // this should reflect all values that are allowed to go through the transport layer
         valueSuppliers.add(() -> null);
-        valueSuppliers.add(() -> randomInt());
-        valueSuppliers.add(() -> randomLong());
-        valueSuppliers.add(() -> randomDouble());
-        valueSuppliers.add(() -> randomFloat());
-        valueSuppliers.add(() -> randomByte());
-        valueSuppliers.add(() -> randomShort());
-        valueSuppliers.add(() -> randomBoolean());
+        valueSuppliers.add(ESTestCase::randomInt);
+        valueSuppliers.add(ESTestCase::randomLong);
+        valueSuppliers.add(ESTestCase::randomDouble);
+        valueSuppliers.add(ESTestCase::randomFloat);
+        valueSuppliers.add(ESTestCase::randomByte);
+        valueSuppliers.add(ESTestCase::randomShort);
+        valueSuppliers.add(ESTestCase::randomBoolean);
         valueSuppliers.add(() -> frequently() ? randomAlphaOfLengthBetween(1, 30) : randomRealisticUnicodeOfCodepointLength(30));
 
         int size = randomIntBetween(1, 20);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
index 06eefb7ccba..68434f0f29e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
@@ -48,7 +48,7 @@ import java.util.List;
 import java.util.Random;
 
 import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
-import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiOfLength;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiLettersOfLength;
 import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomUnicodeOfLengthBetween;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
@@ -122,7 +122,7 @@ public final class RandomObjects {
                     expectedParsedValues.add(randomBoolean);
                     break;
                 case 7:
-                    String randomString = random.nextBoolean() ? RandomStrings.randomAsciiOfLengthBetween(random, 3, 10 ) :
+                    String randomString = random.nextBoolean() ? RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10) :
                             randomUnicodeOfLengthBetween(random, 3, 10);
                     originalValues.add(randomString);
                     expectedParsedValues.add(randomString);
@@ -191,11 +191,11 @@ public final class RandomObjects {
         for (int i = 0; i < numFields; i++) {
             if (currentDepth < 5 && random.nextInt(100) >= 70) {
                 if (random.nextBoolean()) {
-                    builder.startObject(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10));
+                    builder.startObject(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
                     addFields(random, builder, minNumFields, currentDepth + 1);
                     builder.endObject();
                 } else {
-                    builder.startArray(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10));
+                    builder.startArray(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10));
                     int numElements = randomIntBetween(random, 1, 5);
                     boolean object = random.nextBoolean();
                     int dataType = -1;
@@ -214,7 +214,7 @@ public final class RandomObjects {
                     builder.endArray();
                 }
             } else {
-                builder.field(RandomStrings.randomAsciiOfLengthBetween(random, 6, 10),
+                builder.field(RandomStrings.randomAsciiLettersOfLengthBetween(random, 6, 10),
                         randomFieldValue(random, randomDataType(random)));
             }
         }
@@ -227,9 +227,9 @@ public final class RandomObjects {
     private static Object randomFieldValue(Random random, int dataType) {
         switch(dataType) {
             case 0:
-                return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
+                return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
             case 1:
-                return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
+                return RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
             case 2:
                 return random.nextLong();
             case 3:
@@ -287,10 +287,10 @@ public final class RandomObjects {
      * @param random Random generator
      */
     private static Tuple<Failure, Failure> randomShardInfoFailure(Random random) {
-        String index = randomAsciiOfLength(random, 5);
-        String indexUuid = randomAsciiOfLength(random, 5);
+        String index = randomAsciiLettersOfLength(random, 5);
+        String indexUuid = randomAsciiLettersOfLength(random, 5);
         int shardId = randomIntBetween(random, 1, 10);
-        String nodeId = randomAsciiOfLength(random, 5);
+        String nodeId = randomAsciiLettersOfLength(random, 5);
         RestStatus status = randomFrom(random, RestStatus.INTERNAL_SERVER_ERROR, RestStatus.FORBIDDEN, RestStatus.NOT_FOUND);
         boolean primary = random.nextBoolean();
         ShardId shard = new ShardId(index, indexUuid, shardId);

From 0cae19c8d7d373b3138ca5a73adbebf442e2abef Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Mon, 30 Jul 2018 16:24:41 +0300
Subject: [PATCH 222/260] IndicesClusterStateService should replace an init.
 replica with an init. primary with the same aId (#32374)

In rare cases it is possible that a nodes gets an instruction to replace a replica
shard that's in `POST_RECOVERY` with a new initializing primary with the same allocation id.
This can happen by batching cluster states that include the starting of the replica, with
closing of the indices, opening it up again and allocating the primary shard to the node in
question. The node should then clean it's initializing replica and replace it with a new
initializing primary.

I'm not sure whether the test I added really adds enough value as existing tests found this. The main reason I added is to allow for simpler reproduction and to double check I fixed it. I'm open to discuss if we should keep.

Closes #32308
---
 .../cluster/IndicesClusterStateService.java   |  6 ++
 .../ClusterStateCreationUtils.java            | 27 ++++++---
 ...actIndicesClusterStateServiceTestCase.java |  4 ++
 ...ClusterStateServiceRandomUpdatesTests.java | 56 ++++++++++++++++++-
 4 files changed, 84 insertions(+), 9 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 472cb04936d..e6a86d47f55 100644
--- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -420,6 +420,12 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
                     // state may result in a new shard being initialized while having the same allocation id as the currently started shard.
                     logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting);
                     indexService.removeShard(shardId.id(), "removing shard (stale copy)");
+                } else if (newShardRouting.primary() && currentRoutingEntry.primary() == false && newShardRouting.initializing()) {
+                    assert currentRoutingEntry.initializing() : currentRoutingEntry; // see above if clause
+                    // this can happen when cluster state batching batches activation of the shard, closing an index, reopening it
+                    // and assigning an initializing primary to this node
+                    logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting);
+                    indexService.removeShard(shardId.id(), "removing shard (stale copy)");
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index 59ede535c2f..60053748d68 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -27,10 +27,12 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable.Builder;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -44,6 +46,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@@ -93,7 +96,8 @@ public class ClusterStateCreationUtils {
         IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder()
                 .put(SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas)
-                .put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm).build();
+                .put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm)
+            .build();
 
         RoutingTable.Builder routing = new RoutingTable.Builder();
         routing.addAsNew(indexMetaData);
@@ -138,12 +142,19 @@ public class ClusterStateCreationUtils {
                     TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, false, replicaState,
                             unassignedInfo));
         }
+        final IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingBuilder.build();
+
+        IndexMetaData.Builder indexMetaDataBuilder = new IndexMetaData.Builder(indexMetaData);
+        indexMetaDataBuilder.putInSyncAllocationIds(0,
+            indexShardRoutingTable.activeShards().stream().map(ShardRouting::allocationId).map(AllocationId::getId)
+            .collect(Collectors.toSet())
+        );
 
         ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
         state.nodes(discoBuilder);
-        state.metaData(MetaData.builder().put(indexMetaData, false).generateClusterUuidIfNeeded());
+        state.metaData(MetaData.builder().put(indexMetaDataBuilder.build(), false).generateClusterUuidIfNeeded());
         state.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder(indexMetaData.getIndex())
-                .addIndexShard(indexShardRoutingBuilder.build())).build());
+                .addIndexShard(indexShardRoutingTable)).build());
         return state.build();
     }
 
@@ -272,21 +283,21 @@ public class ClusterStateCreationUtils {
         state.routingTable(RoutingTable.builder().add(indexRoutingTableBuilder.build()).build());
         return state.build();
     }
-    
-    
+
+
     /**
      * Creates cluster state with several indexes, shards and replicas and all shards STARTED.
      */
     public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indices, int numberOfShards, int numberOfReplicas) {
 
-        int numberOfDataNodes = numberOfReplicas + 1; 
+        int numberOfDataNodes = numberOfReplicas + 1;
         DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
         for (int i = 0; i < numberOfDataNodes + 1; i++) {
             final DiscoveryNode node = newNode(i);
             discoBuilder = discoBuilder.add(node);
         }
         discoBuilder.localNodeId(newNode(0).getId());
-        discoBuilder.masterNodeId(newNode(numberOfDataNodes + 1).getId()); 
+        discoBuilder.masterNodeId(newNode(numberOfDataNodes + 1).getId());
         ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
         state.nodes(discoBuilder);
         Builder routingTableBuilder = RoutingTable.builder();
@@ -316,7 +327,7 @@ public class ClusterStateCreationUtils {
         state.metaData(metadataBuilder);
         state.routingTable(routingTableBuilder.build());
         return state.build();
-    }    
+    }
 
     /**
      * Creates cluster state with and index that has one shard and as many replicas as numberOfReplicas.
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
index 5c6b000f7e5..580696264bd 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
@@ -74,6 +74,10 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC
         enableRandomFailures = randomBoolean();
     }
 
+    protected void disableRandomFailures() {
+        enableRandomFailures = false;
+    }
+
     protected void failRandomly() {
         if (enableRandomFailures && rarely()) {
             throw new RuntimeException("dummy test failure");
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 5611421594a..39091ce04ec 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -49,6 +49,7 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -75,6 +76,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_AUTO_EXPA
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -97,7 +99,6 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
         terminate(threadPool);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32308")
     public void testRandomClusterStateUpdates() {
         // we have an IndicesClusterStateService per node in the cluster
         final Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap = new HashMap<>();
@@ -199,6 +200,59 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
         }
     }
 
+    /**
+     * In rare cases it is possible that a nodes gets an instruction to replace a replica
+     * shard that's in POST_RECOVERY with a new initializing primary with the same allocation id.
+     * This can happen by batching cluster states that include the starting of the replica, with
+     * closing of the indices, opening it up again and allocating the primary shard to the node in
+     * question. The node should then clean it's initializing replica and replace it with a new
+     * initializing primary.
+     */
+    public void testInitializingPrimaryRemovesInitializingReplicaWithSameAID() {
+        disableRandomFailures();
+        String index = "index_" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
+        ClusterState state = ClusterStateCreationUtils.state(index, randomBoolean(),
+            ShardRoutingState.STARTED, ShardRoutingState.INITIALIZING);
+
+        // the initial state which is derived from the newly created cluster state but doesn't contain the index
+        ClusterState previousState = ClusterState.builder(state)
+            .metaData(MetaData.builder(state.metaData()).remove(index))
+            .routingTable(RoutingTable.builder().build())
+            .build();
+
+        // pick a data node to simulate the adding an index cluster state change event on, that has shards assigned to it
+        final ShardRouting shardRouting = state.routingTable().index(index).shard(0).replicaShards().get(0);
+        final ShardId shardId = shardRouting.shardId();
+        DiscoveryNode node = state.nodes().get(shardRouting.currentNodeId());
+
+        // simulate the cluster state change on the node
+        ClusterState localState = adaptClusterStateToLocalNode(state, node);
+        ClusterState previousLocalState = adaptClusterStateToLocalNode(previousState, node);
+        IndicesClusterStateService indicesCSSvc = createIndicesClusterStateService(node, RecordingIndicesService::new);
+        indicesCSSvc.start();
+        indicesCSSvc.applyClusterState(new ClusterChangedEvent("cluster state change that adds the index", localState, previousLocalState));
+        previousState = state;
+
+        // start the replica
+        state = cluster.applyStartedShards(state, state.routingTable().index(index).shard(0).replicaShards());
+
+        // close the index and open it up again (this will sometimes swap roles between primary and replica)
+        CloseIndexRequest closeIndexRequest = new CloseIndexRequest(state.metaData().index(index).getIndex().getName());
+        state = cluster.closeIndices(state, closeIndexRequest);
+        OpenIndexRequest openIndexRequest = new OpenIndexRequest(state.metaData().index(index).getIndex().getName());
+        state = cluster.openIndices(state, openIndexRequest);
+
+        localState = adaptClusterStateToLocalNode(state, node);
+        previousLocalState = adaptClusterStateToLocalNode(previousState, node);
+
+        indicesCSSvc.applyClusterState(new ClusterChangedEvent("new cluster state", localState, previousLocalState));
+
+        final MockIndexShard shardOrNull = ((RecordingIndicesService) indicesCSSvc.indicesService).getShardOrNull(shardId);
+        assertThat(shardOrNull == null ? null : shardOrNull.routingEntry(),
+            equalTo(state.getRoutingNodes().node(node.getId()).getByShardId(shardId)));
+
+    }
+
     public ClusterState randomInitialClusterState(Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap,
                                                   Supplier<MockIndicesService> indicesServiceSupplier) {
         List<DiscoveryNode> allNodes = new ArrayList<>();

From 34d006f82a3061f56c76266be2971a7db3bde4c6 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 30 Jul 2018 10:00:55 -0700
Subject: [PATCH 223/260] Tests: Fix convert error tests to use fixed value
 (#32415)

The error tests for hex values previously used a random string of
digits, but this could be a valid hex value. This commit changes these
tests to use a fixed invalid hex value.

closes #32370
---
 .../elasticsearch/ingest/common/ConvertProcessorTests.java  | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
index 500f1aa1719..34fd4ec27a3 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
@@ -67,10 +67,9 @@ public class ConvertProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(10));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
     public void testConvertIntHexError() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
-        String value = "0x" + randomAlphaOfLengthBetween(1, 10);
+        String value = "0xnotanumber";
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value);
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
@@ -138,10 +137,9 @@ public class ConvertProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L));
     }
 
-    @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
     public void testConvertLongHexError() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
-        String value = "0x" + randomAlphaOfLengthBetween(1, 10);
+        String value = "0xnotanumber";
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value);
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));

From c2e3bebab997367520d18761288ffea5a9a715f1 Mon Sep 17 00:00:00 2001
From: Ioannis Kakavas <ikakavas@protonmail.com>
Date: Mon, 30 Jul 2018 22:15:59 +0300
Subject: [PATCH 224/260] Ensure KeyStoreWrapper decryption exceptions are
 handled (#32464)

* Ensure decryption related exceptions are handled

This commit ensures that all possible Exceptions in
KeyStoreWrapper#decrypt() are handled. More specifically, in the
case that a wrong password is used for secure settings, calling readX
on the DataInputStream that wraps the CipherInputStream can throw an
IOException. It also adds a test for loading a KeyStoreWrapper with
a wrong password.

Resolves #32411
---
 .../elasticsearch/common/settings/KeyStoreWrapper.java   | 2 +-
 .../action/admin/ReloadSecureSettingsIT.java             | 8 --------
 .../common/settings/KeyStoreWrapperTests.java            | 9 +++++++++
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
index eee45743ee3..e017e9e7ca9 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -359,7 +359,7 @@ public class KeyStoreWrapper implements SecureSettings {
             if (input.read() != -1) {
                 throw new SecurityException("Keystore has been corrupted or tampered with");
             }
-        } catch (EOFException e) {
+        } catch (IOException e) {
             throw new SecurityException("Keystore has been corrupted or tampered with", e);
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
index c8503603f66..79527582405 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java
@@ -32,7 +32,6 @@ import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
@@ -205,14 +204,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
                             assertThat(nodesMap.size(), equalTo(cluster().size()));
                             for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) {
                                 assertThat(nodeResponse.reloadException(), notNullValue());
-                                // Running in a JVM with a BouncyCastle FIPS Security Provider, decrypting the Keystore with the wrong
-                                // password returns a SecurityException if the DataInputStream can't be fully consumed
-                                if (inFipsJvm()) {
                                     assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class));
-                                } else {
-                                    assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
-                                }
-
                             }
                         } catch (final AssertionError e) {
                             reloadSettingsError.set(e);
diff --git a/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
index fe7b02d63ec..bb2b1df7f8c 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
@@ -99,6 +99,15 @@ public class KeyStoreWrapperTests extends ESTestCase {
         assertTrue(keystore.getSettingNames().contains(KeyStoreWrapper.SEED_SETTING.getKey()));
     }
 
+    public void testDecryptKeyStoreWithWrongPassword() throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.create();
+        keystore.save(env.configFile(), new char[0]);
+        final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configFile());
+        final SecurityException exception = expectThrows(SecurityException.class,
+            () -> loadedkeystore.decrypt(new char[]{'i', 'n', 'v', 'a', 'l', 'i', 'd'}));
+        assertThat(exception.getMessage(), containsString("Keystore has been corrupted or tampered with"));
+    }
+
     public void testCannotReadStringFromClosedKeystore() throws Exception {
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
         assertThat(keystore.getSettingNames(), Matchers.hasItem(KeyStoreWrapper.SEED_SETTING.getKey()));

From cf7489899a957764404769a4b792cdbefbf6a729 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 30 Jul 2018 21:25:30 +0200
Subject: [PATCH 225/260] INGEST: Clean up Java8 Stream Usage (#32059)

* GrokProcessor: Rationalize the loop over the map to save allocations and indirection
* IngestDocument: Rationalize way we append to `List`
---
 .../java/org/elasticsearch/ingest/common/GrokProcessor.java    | 3 +--
 .../src/main/java/org/elasticsearch/ingest/IngestDocument.java | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
index 7bb3ebfba6e..88cba512b86 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
@@ -68,8 +68,7 @@ public final class GrokProcessor extends AbstractProcessor {
             throw new IllegalArgumentException("Provided Grok expressions do not match field value: [" + fieldValue + "]");
         }
 
-        matches.entrySet().stream()
-            .forEach((e) -> ingestDocument.setFieldValue(e.getKey(), e.getValue()));
+        matches.forEach(ingestDocument::setFieldValue);
 
         if (traceMatch) {
             if (matchPatterns.size() > 1) {
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
index 2bd842e72b1..aad55e12cef 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -527,8 +527,7 @@ public final class IngestDocument {
 
     private static void appendValues(List<Object> list, Object value) {
         if (value instanceof List) {
-            List<?> valueList = (List<?>) value;
-            valueList.stream().forEach(list::add);
+            list.addAll((List<?>) value);
         } else {
             list.add(value);
         }

From 072c0be8af0bdbe9e035e4da6bd8e3831fe1796a Mon Sep 17 00:00:00 2001
From: w-bonelli <wpb3hw@virginia.edu>
Date: Mon, 30 Jul 2018 15:48:14 -0400
Subject: [PATCH 226/260] Update Fuzzy Query docs to clarify default behavior
 re max_expansions (#30819)

Stating that the Fuzzy Query generates "all possible" matching terms is misleading, given that the query's default behavior is to generate a maximum of 50 matching terms.

(cherry picked from commit 345a0071a2a41fd7f80ae9ef8a39a2cb4991aedd)
---
 docs/reference/query-dsl/fuzzy-query.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc
index fa11cb257d4..4be54691624 100644
--- a/docs/reference/query-dsl/fuzzy-query.asciidoc
+++ b/docs/reference/query-dsl/fuzzy-query.asciidoc
@@ -5,10 +5,10 @@ The fuzzy query uses similarity based on Levenshtein edit distance.
 
 ==== String fields
 
-The `fuzzy` query generates all possible matching terms that are within  the
+The `fuzzy` query generates matching terms that are within the
 maximum edit distance specified in `fuzziness` and then checks the term
 dictionary to find out which of those generated terms actually exist in the
-index.
+index. The final query uses up to `max_expansions` matching terms.
 
 Here is a simple example:
 

From 1e0fcebfe1227401343e879cb07fd212c20fe99e Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Mon, 30 Jul 2018 14:32:55 -0700
Subject: [PATCH 227/260] update rollover to leverage write-alias semantics
 (#32216)

Rollover should not swap aliases when `is_write_index` is set to `true`.
Instead, both the new and old indices should have the rollover alias,
with the newly created index as the new write index

Updates Rollover to leverage the ability to preserve aliases and swap which is the write index.

Historically, Rollover would swap which index had the designated alias for writing documents against. This required users to keep a separate read-alias that enabled reading against both rolled over and newly created indices, whiles the write-alias was being re-assigned at every rollover.

With the ability for aliases to designate a write index, Rollover can be a bit more flexible with its use of aliases.

Updates include:

- Rollover validates that the target alias has a write index (the index that is being rolled over). This means that the restriction that aliases only point to one index is no longer necessary.
- Rollover explicitly (and atomically) swaps which index is the write-index by explicitly assigning the existing index to have `is_write_index: false` and have the newly created index have its rollover alias as `is_write_index: true`. This is only done when `is_write_index: true` on the write index. Default behavior of removing the alias from the rolled over index stays when `is_write_index` is not explicitly set

Relevant things that are staying the same:

- Rollover is rejected if there exist any templates that match the newly-created index and configure the rollover-alias
   - I think this existed to prevent the situation where an alias pointed to two indices for a short while. Although this can technically be relaxed, the specific cases that are safe are really particular and difficult to reason, so leaving the broad restriction sounds good
---
 docs/reference/indices/aliases.asciidoc       |   3 +
 .../reference/indices/rollover-index.asciidoc | 112 +++++++++++++++++-
 .../rollover/TransportRolloverAction.java     |  38 ++++--
 .../admin/indices/rollover/RolloverIT.java    |  90 ++++++++++++--
 .../TransportRolloverActionTests.java         |  68 ++++++++---
 5 files changed, 274 insertions(+), 37 deletions(-)

diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc
index 8e0e84c6657..81a96510e46 100644
--- a/docs/reference/indices/aliases.asciidoc
+++ b/docs/reference/indices/aliases.asciidoc
@@ -257,6 +257,9 @@ and there are multiple indices referenced by an alias, then writes will not be a
 It is possible to specify an index associated with an alias as a write index using both the aliases API
 and index creation API.
 
+Setting an index to be the write index with an alias also affects how the alias is manipulated during
+Rollover (see <<indices-rollover-index, Rollover With Write Index>>).
+
 [source,js]
 --------------------------------------------------
 POST /_aliases
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 1e3a361f1b1..1730c95e0dd 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -4,10 +4,19 @@
 The rollover index API rolls an alias over to a new index when the existing
 index is considered to be too large or too old.
 
-The API accepts a single alias name and a list of `conditions`.  The alias
-must point to a single index only.  If the index satisfies the specified
-conditions then a new index is created and the alias is switched to point to
-the new index.
+The API accepts a single alias name and a list of `conditions`. The alias must point to a write index for
+a Rollover request to be valid. There are two ways this can be achieved, and depending on the configuration, the
+alias metadata will be updated differently. The two scenarios are as follows:
+
+ - The alias only points to a single index with `is_write_index` not configured (defaults to `null`).
+
+In this scenario, the original index will have their rollover alias will be added to the newly created index, and removed
+from the original (rolled-over) index.
+
+ - The alias points to one or more indices with `is_write_index` set to `true` on the index to be rolled over (the write index).
+
+In this scenario, the write index will have its rollover alias' `is_write_index` set to `false`, while the newly created index
+will now have the rollover alias pointing to it as the write index with `is_write_index` as `true`.
 
 
 [source,js]
@@ -231,3 +240,98 @@ POST /logs_write/_rollover?dry_run
 Because the rollover operation creates a new index to rollover to, the
 <<create-index-wait-for-active-shards,`wait_for_active_shards`>> setting on
 index creation applies to the rollover action as well.
+
+[[indices-rollover-is-write-index]]
+[float]
+=== Write Index Alias Behavior
+
+The rollover alias when rolling over a write index that has `is_write_index` explicitly set to `true` is not
+swapped during rollover actions. Since having an alias point to multiple indices is ambiguous in distinguishing
+which is the correct write index to roll over, it is not valid to rollover an alias that points to multiple indices.
+For this reason, the default behavior is to swap which index is being pointed to by the write-oriented alias. This
+was `logs_write` in some of the above examples. Since setting `is_write_index` enables an alias to point to multiple indices
+while also being explicit as to which is the write index that rollover should target, removing the alias from the rolled over
+index is not necessary. This simplifies things by allowing for one alias to behave both as the write and read aliases for
+indices that are being managed with Rollover.
+
+Look at the behavior of the aliases in the following example where `is_write_index` is set on the rolled over index.
+
+[source,js]
+--------------------------------------------------
+PUT my_logs_index-000001
+{
+  "aliases": {
+    "logs": { "is_write_index": true } <1>
+  }
+}
+
+PUT logs/_doc/1
+{
+  "message": "a dummy log"
+}
+
+POST logs/_refresh
+
+POST /logs/_rollover
+{
+  "conditions": {
+    "max_docs":   "1"
+  }
+}
+
+PUT logs/_doc/2 <2>
+{
+  "message": "a newer log"
+}
+--------------------------------------------------
+// CONSOLE
+<1> configures `my_logs_index` as the write index for the `logs` alias
+<2> newly indexed documents against the `logs` alias will write to the new index
+
+[source,js]
+--------------------------------------------------
+{
+  "_index" : "my_logs_index-000002",
+  "_type" : "_doc",
+  "_id" : "2",
+  "_version" : 1,
+  "result" : "created",
+  "_shards" : {
+    "total" : 2,
+    "successful" : 1,
+    "failed" : 0
+  },
+  "_seq_no" : 0,
+  "_primary_term" : 1
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+//////////////////////////
+[source,js]
+--------------------------------------------------
+GET _alias
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+//////////////////////////
+
+After the rollover, the alias metadata for the two indices will have the `is_write_index` setting
+reflect each index's role, with the newly created index as the write index.
+
+[source,js]
+--------------------------------------------------
+{
+  "my_logs_index-000002": {
+    "aliases": {
+      "logs": { "is_write_index": true }
+    }
+  },
+  "my_logs_index-000001": {
+    "aliases": {
+      "logs": { "is_write_index" : false }
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 3fa046263af..ab05d069003 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -109,8 +109,9 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                                    final ActionListener<RolloverResponse> listener) {
         final MetaData metaData = state.metaData();
         validate(metaData, rolloverRequest);
-        final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getAlias());
-        final IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
+        final AliasOrIndex.Alias alias = (AliasOrIndex.Alias) metaData.getAliasAndIndexLookup().get(rolloverRequest.getAlias());
+        final IndexMetaData indexMetaData = alias.getWriteIndex();
+        final boolean explicitWriteIndex = Boolean.TRUE.equals(indexMetaData.getAliases().get(alias.getAliasName()).writeIndex());
         final String sourceProvidedName = indexMetaData.getSettings().get(IndexMetaData.SETTING_INDEX_PROVIDED_NAME,
             indexMetaData.getIndex().getName());
         final String sourceIndexName = indexMetaData.getIndex().getName();
@@ -138,10 +139,15 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                         CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(unresolvedName, rolloverIndexName,
                             rolloverRequest);
                         createIndexService.createIndex(updateRequest, ActionListener.wrap(createIndexClusterStateUpdateResponse -> {
-                            // switch the alias to point to the newly created index
-                            indexAliasesService.indicesAliases(
-                                prepareRolloverAliasesUpdateRequest(sourceIndexName, rolloverIndexName,
-                                    rolloverRequest),
+                            final IndicesAliasesClusterStateUpdateRequest aliasesUpdateRequest;
+                            if (explicitWriteIndex) {
+                                aliasesUpdateRequest = prepareRolloverAliasesWriteIndexUpdateRequest(sourceIndexName,
+                                    rolloverIndexName, rolloverRequest);
+                            } else {
+                                aliasesUpdateRequest = prepareRolloverAliasesUpdateRequest(sourceIndexName,
+                                    rolloverIndexName, rolloverRequest);
+                            }
+                            indexAliasesService.indicesAliases(aliasesUpdateRequest,
                                 ActionListener.wrap(aliasClusterStateUpdateResponse -> {
                                     if (aliasClusterStateUpdateResponse.isAcknowledged()) {
                                         clusterService.submitStateUpdateTask("update_rollover_info", new ClusterStateUpdateTask() {
@@ -196,8 +202,19 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
     static IndicesAliasesClusterStateUpdateRequest prepareRolloverAliasesUpdateRequest(String oldIndex, String newIndex,
                                                                                        RolloverRequest request) {
         List<AliasAction> actions = unmodifiableList(Arrays.asList(
-                new AliasAction.Add(newIndex, request.getAlias(), null, null, null, null),
-                new AliasAction.Remove(oldIndex, request.getAlias())));
+            new AliasAction.Add(newIndex, request.getAlias(), null, null, null, null),
+            new AliasAction.Remove(oldIndex, request.getAlias())));
+        final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest(actions)
+            .ackTimeout(request.ackTimeout())
+            .masterNodeTimeout(request.masterNodeTimeout());
+        return updateRequest;
+    }
+
+    static IndicesAliasesClusterStateUpdateRequest prepareRolloverAliasesWriteIndexUpdateRequest(String oldIndex, String newIndex,
+                                                                                                 RolloverRequest request) {
+        List<AliasAction> actions = unmodifiableList(Arrays.asList(
+            new AliasAction.Add(newIndex, request.getAlias(), null, null, null, true),
+            new AliasAction.Add(oldIndex, request.getAlias(), null, null, null, false)));
         final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest(actions)
             .ackTimeout(request.ackTimeout())
             .masterNodeTimeout(request.masterNodeTimeout());
@@ -244,8 +261,9 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         if (aliasOrIndex.isAlias() == false) {
             throw new IllegalArgumentException("source alias is a concrete index");
         }
-        if (aliasOrIndex.getIndices().size() != 1) {
-            throw new IllegalArgumentException("source alias maps to multiple indices");
+        final AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex;
+        if (alias.getWriteIndex() == null) {
+            throw new IllegalArgumentException("source alias [" + alias.getAliasName() + "] does not point to a write index");
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 5379769e819..7823cd8849c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -60,7 +60,12 @@ public class RolloverIT extends ESIntegTestCase {
 
 
     public void testRolloverOnEmptyIndex() throws Exception {
-        assertAcked(prepareCreate("test_index-1").addAlias(new Alias("test_alias")).get());
+        Alias testAlias = new Alias("test_alias");
+        boolean explicitWriteIndex = randomBoolean();
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index-1").addAlias(testAlias).get());
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
         assertThat(response.getOldIndex(), equalTo("test_index-1"));
         assertThat(response.getNewIndex(), equalTo("test_index-000002"));
@@ -69,7 +74,12 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-1");
-        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        if (explicitWriteIndex) {
+            assertTrue(oldIndex.getAliases().containsKey("test_alias"));
+            assertFalse(oldIndex.getAliases().get("test_alias").writeIndex());
+        } else {
+            assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        }
         final IndexMetaData newIndex = state.metaData().index("test_index-000002");
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
     }
@@ -97,8 +107,49 @@ public class RolloverIT extends ESIntegTestCase {
             is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L))));
     }
 
+    public void testRolloverWithExplicitWriteIndex() throws Exception {
+        long beforeTime = client().threadPool().absoluteTimeInMillis() - 1000L;
+        assertAcked(prepareCreate("test_index-2").addAlias(new Alias("test_alias").writeIndex(true)).get());
+        index("test_index-2", "type1", "1", "field", "value");
+        flush("test_index-2");
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
+        assertThat(response.getOldIndex(), equalTo("test_index-2"));
+        assertThat(response.getNewIndex(), equalTo("test_index-000003"));
+        assertThat(response.isDryRun(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index-2");
+        assertTrue(oldIndex.getAliases().containsKey("test_alias"));
+        assertFalse(oldIndex.getAliases().get("test_alias").writeIndex());
+        final IndexMetaData newIndex = state.metaData().index("test_index-000003");
+        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+        assertTrue(newIndex.getAliases().get("test_alias").writeIndex());
+        assertThat(oldIndex.getRolloverInfos().size(), equalTo(1));
+        assertThat(oldIndex.getRolloverInfos().get("test_alias").getAlias(), equalTo("test_alias"));
+        assertThat(oldIndex.getRolloverInfos().get("test_alias").getMetConditions(), is(empty()));
+        assertThat(oldIndex.getRolloverInfos().get("test_alias").getTime(),
+            is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L))));
+    }
+
+    public void testRolloverWithNoWriteIndex() {
+        Boolean firstIsWriteIndex = randomFrom(false, null);
+        assertAcked(prepareCreate("index1").addAlias(new Alias("alias").writeIndex(firstIsWriteIndex)).get());
+        if (firstIsWriteIndex == null) {
+            assertAcked(prepareCreate("index2").addAlias(new Alias("alias").writeIndex(randomFrom(false, null))).get());
+        }
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> client().admin().indices().prepareRolloverIndex("alias").dryRun(randomBoolean()).get());
+        assertThat(exception.getMessage(), equalTo("source alias [alias] does not point to a write index"));
+    }
+
     public void testRolloverWithIndexSettings() throws Exception {
-        assertAcked(prepareCreate("test_index-2").addAlias(new Alias("test_alias")).get());
+        Alias testAlias = new Alias("test_alias");
+        boolean explicitWriteIndex = randomBoolean();
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index-2").addAlias(testAlias).get());
         index("test_index-2", "type1", "1", "field", "value");
         flush("test_index-2");
         final Settings settings = Settings.builder()
@@ -114,12 +165,17 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-2");
-        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
         final IndexMetaData newIndex = state.metaData().index("test_index-000003");
         assertThat(newIndex.getNumberOfShards(), equalTo(1));
         assertThat(newIndex.getNumberOfReplicas(), equalTo(0));
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
         assertTrue(newIndex.getAliases().containsKey("extra_alias"));
+        if (explicitWriteIndex) {
+            assertFalse(oldIndex.getAliases().get("test_alias").writeIndex());
+            assertTrue(newIndex.getAliases().get("test_alias").writeIndex());
+        } else {
+            assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        }
     }
 
     public void testRolloverDryRun() throws Exception {
@@ -140,7 +196,12 @@ public class RolloverIT extends ESIntegTestCase {
     }
 
     public void testRolloverConditionsNotMet() throws Exception {
-        assertAcked(prepareCreate("test_index-0").addAlias(new Alias("test_alias")).get());
+        boolean explicitWriteIndex = randomBoolean();
+        Alias testAlias = new Alias("test_alias");
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index-0").addAlias(testAlias).get());
         index("test_index-0", "type1", "1", "field", "value");
         flush("test_index-0");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
@@ -160,12 +221,22 @@ public class RolloverIT extends ESIntegTestCase {
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-0");
         assertTrue(oldIndex.getAliases().containsKey("test_alias"));
+        if (explicitWriteIndex) {
+            assertTrue(oldIndex.getAliases().get("test_alias").writeIndex());
+        } else {
+            assertNull(oldIndex.getAliases().get("test_alias").writeIndex());
+        }
         final IndexMetaData newIndex = state.metaData().index("test_index-000001");
         assertNull(newIndex);
     }
 
     public void testRolloverWithNewIndexName() throws Exception {
-        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        Alias testAlias = new Alias("test_alias");
+        boolean explicitWriteIndex = randomBoolean();
+        if (explicitWriteIndex) {
+            testAlias.writeIndex(true);
+        }
+        assertAcked(prepareCreate("test_index").addAlias(testAlias).get());
         index("test_index", "type1", "1", "field", "value");
         flush("test_index");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
@@ -177,9 +248,14 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index");
-        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
         final IndexMetaData newIndex = state.metaData().index("test_new_index");
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
+        if (explicitWriteIndex) {
+            assertFalse(oldIndex.getAliases().get("test_alias").writeIndex());
+            assertTrue(newIndex.getAliases().get("test_alias").writeIndex());
+        } else {
+            assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        }
     }
 
     public void testRolloverOnExistingIndex() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 6149c380cd7..d59700f2b7a 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -172,39 +172,75 @@ public class TransportRolloverActionTests extends ESTestCase {
         assertTrue(foundRemove);
     }
 
+    public void testCreateUpdateAliasRequestWithExplicitWriteIndex() {
+        String sourceAlias = randomAlphaOfLength(10);
+        String sourceIndex = randomAlphaOfLength(10);
+        String targetIndex = randomAlphaOfLength(10);
+        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias, targetIndex);
+        final IndicesAliasesClusterStateUpdateRequest updateRequest =
+            TransportRolloverAction.prepareRolloverAliasesWriteIndexUpdateRequest(sourceIndex, targetIndex, rolloverRequest);
+
+        List<AliasAction> actions = updateRequest.actions();
+        assertThat(actions, hasSize(2));
+        boolean foundAddWrite = false;
+        boolean foundRemoveWrite = false;
+        for (AliasAction action : actions) {
+            AliasAction.Add addAction = (AliasAction.Add) action;
+            if (action.getIndex().equals(targetIndex)) {
+                assertEquals(sourceAlias, addAction.getAlias());
+                assertTrue(addAction.writeIndex());
+                foundAddWrite = true;
+            } else if (action.getIndex().equals(sourceIndex)) {
+                assertEquals(sourceAlias, addAction.getAlias());
+                assertFalse(addAction.writeIndex());
+                foundRemoveWrite = true;
+            } else {
+                throw new AssertionError("Unknow index [" + action.getIndex() + "]");
+            }
+        }
+        assertTrue(foundAddWrite);
+        assertTrue(foundRemoveWrite);
+    }
+
     public void testValidation() {
         String index1 = randomAlphaOfLength(10);
-        String alias = randomAlphaOfLength(10);
+        String aliasWithWriteIndex = randomAlphaOfLength(10);
         String index2 = randomAlphaOfLength(10);
-        String aliasWithMultipleIndices = randomAlphaOfLength(10);
+        String aliasWithNoWriteIndex = randomAlphaOfLength(10);
+        Boolean firstIsWriteIndex = randomFrom(false, null);
         final Settings settings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
             .build();
-        final MetaData metaData = MetaData.builder()
+        MetaData.Builder metaDataBuilder = MetaData.builder()
             .put(IndexMetaData.builder(index1)
                 .settings(settings)
-                .putAlias(AliasMetaData.builder(alias))
-                .putAlias(AliasMetaData.builder(aliasWithMultipleIndices))
-            )
-            .put(IndexMetaData.builder(index2)
-                .settings(settings)
-                .putAlias(AliasMetaData.builder(aliasWithMultipleIndices))
-            ).build();
+                .putAlias(AliasMetaData.builder(aliasWithWriteIndex))
+                .putAlias(AliasMetaData.builder(aliasWithNoWriteIndex).writeIndex(firstIsWriteIndex))
+            );
+        IndexMetaData.Builder indexTwoBuilder = IndexMetaData.builder(index2).settings(settings);
+        if (firstIsWriteIndex == null) {
+            indexTwoBuilder.putAlias(AliasMetaData.builder(aliasWithNoWriteIndex).writeIndex(randomFrom(false, null)));
+        }
+        metaDataBuilder.put(indexTwoBuilder);
+        MetaData metaData = metaDataBuilder.build();
 
-        expectThrows(IllegalArgumentException.class, () ->
-            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices,
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithNoWriteIndex,
                 randomAlphaOfLength(10))));
-        expectThrows(IllegalArgumentException.class, () ->
+        assertThat(exception.getMessage(), equalTo("source alias [" + aliasWithNoWriteIndex + "] does not point to a write index"));
+        exception = expectThrows(IllegalArgumentException.class, () ->
             TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2),
                 randomAlphaOfLength(10))));
-        expectThrows(IllegalArgumentException.class, () ->
+        assertThat(exception.getMessage(), equalTo("source alias is a concrete index"));
+        exception = expectThrows(IllegalArgumentException.class, () ->
             TransportRolloverAction.validate(metaData, new RolloverRequest(randomAlphaOfLength(5),
                 randomAlphaOfLength(10)))
         );
-        TransportRolloverAction.validate(metaData, new RolloverRequest(alias, randomAlphaOfLength(10)));
+        assertThat(exception.getMessage(), equalTo("source alias does not exist"));
+        TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithWriteIndex, randomAlphaOfLength(10)));
     }
 
     public void testGenerateRolloverIndexName() {
@@ -248,7 +284,7 @@ public class TransportRolloverActionTests extends ESTestCase {
     public void testRejectDuplicateAlias() {
         final IndexTemplateMetaData template = IndexTemplateMetaData.builder("test-template")
             .patterns(Arrays.asList("foo-*", "bar-*"))
-            .putAlias(AliasMetaData.builder("foo-write")).putAlias(AliasMetaData.builder("bar-write"))
+            .putAlias(AliasMetaData.builder("foo-write")).putAlias(AliasMetaData.builder("bar-write").writeIndex(randomBoolean()))
             .build();
         final MetaData metaData = MetaData.builder().put(createMetaData(), false).put(template).build();
         String indexName = randomFrom("foo-123", "bar-xyz");

From c69e62d96fef43e526bd130acc429a5a8536cc56 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 30 Jul 2018 14:46:24 -0700
Subject: [PATCH 228/260] Painless: Add PainlessConstructor (#32447)

PainlessMethod was being used as both a method and a constructor, and while there are
similarities, there are also some major differences. This allows the reflection objects to be
stored reducing the number of other pieces of data stored in a PainlessMethod as they are
now redundant. This temporarily increases some of the code in FunctionRef and
PainlessDocGenerator as they now differentiate between constructors and methods, BUT
is also makes the code more maintainable because there aren't checks in several places
anymore to differentiate.
---
 .../java/org/elasticsearch/painless/Def.java  |   2 +-
 .../elasticsearch/painless/FunctionRef.java   | 119 +++++++++++++-----
 .../painless/lookup/PainlessClass.java        |   6 +-
 .../painless/lookup/PainlessClassBuilder.java |   4 +-
 .../painless/lookup/PainlessConstructor.java  |  39 ++++++
 .../lookup/PainlessLookupBuilder.java         |  59 ++++++---
 .../lookup/PainlessLookupUtility.java         |   7 ++
 .../painless/node/ECapturingFunctionRef.java  |   6 +-
 .../painless/node/EFunctionRef.java           |   2 +-
 .../painless/node/EListInit.java              |  11 +-
 .../elasticsearch/painless/node/EMapInit.java |  11 +-
 .../elasticsearch/painless/node/ENewObj.java  |  18 +--
 .../painless/PainlessDocGenerator.java        |  99 +++++++++++++--
 13 files changed, 307 insertions(+), 76 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 8a90f53b4fd..9be2559e67a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -368,7 +368,7 @@ public final class Def {
              ref = new FunctionRef(clazz, interfaceMethod, call, handle.type(), captures.length);
          } else {
              // whitelist lookup
-             ref = new FunctionRef(painlessLookup, clazz, type, call, captures.length);
+             ref = FunctionRef.resolveFromLookup(painlessLookup, clazz, type, call, captures.length);
          }
          final CallSite callSite = LambdaBootstrap.lambdaBootstrap(
              methodHandlesLookup,
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 7b7e8d25f39..0da91438326 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -20,13 +20,16 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.objectweb.asm.Type;
 
 import java.lang.invoke.MethodType;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Modifier;
+import java.util.List;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
 import static org.objectweb.asm.Opcodes.H_INVOKEINTERFACE;
@@ -59,8 +62,10 @@ public class FunctionRef {
 
     /** interface method */
     public final PainlessMethod interfaceMethod;
-    /** delegate method */
-    public final PainlessMethod delegateMethod;
+    /** delegate method type parameters */
+    public final List<Class<?>> delegateTypeParameters;
+    /** delegate method return type */
+    public final Class<?> delegateReturnType;
 
     /** factory method type descriptor */
     public final String factoryDescriptor;
@@ -80,9 +85,47 @@ public class FunctionRef {
      * @param call the right hand side of a method reference expression
      * @param numCaptures number of captured arguments
      */
-    public FunctionRef(PainlessLookup painlessLookup, Class<?> expected, String type, String call, int numCaptures) {
-        this(expected, painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod,
-                lookup(painlessLookup, expected, type, call, numCaptures > 0), numCaptures);
+    public static FunctionRef resolveFromLookup(
+            PainlessLookup painlessLookup, Class<?> expected, String type, String call, int numCaptures) {
+
+        if ("new".equals(call)) {
+            return new FunctionRef(expected, painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod,
+                    lookup(painlessLookup, expected, type), numCaptures);
+        } else {
+            return new FunctionRef(expected, painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod,
+                    lookup(painlessLookup, expected, type, call, numCaptures > 0), numCaptures);
+        }
+    }
+
+    /**
+     * Creates a new FunctionRef (already resolved)
+     * @param expected functional interface type to implement
+     * @param interfaceMethod functional interface method
+     * @param delegateConstructor implementation constructor
+     * @param numCaptures number of captured arguments
+     */
+    public FunctionRef(Class<?> expected, PainlessMethod interfaceMethod, PainlessConstructor delegateConstructor, int numCaptures) {
+        Constructor<?> javaConstructor = delegateConstructor.javaConstructor;
+        MethodType delegateMethodType = delegateConstructor.methodType;
+
+        interfaceMethodName = interfaceMethod.name;
+        factoryMethodType = MethodType.methodType(expected,
+                delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
+        interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
+
+        delegateClassName = javaConstructor.getDeclaringClass().getName();
+        isDelegateInterface = false;
+        delegateInvokeType = H_NEWINVOKESPECIAL;
+        delegateMethodName = PainlessLookupUtility.CONSTRUCTOR_NAME;
+        this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
+
+        this.interfaceMethod = interfaceMethod;
+        delegateTypeParameters = delegateConstructor.typeParameters;
+        delegateReturnType = void.class;
+
+        factoryDescriptor = factoryMethodType.toMethodDescriptorString();
+        interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
+        delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString());
     }
 
     /**
@@ -112,9 +155,7 @@ public class FunctionRef {
             isDelegateInterface = delegateMethod.target.isInterface();
         }
 
-        if ("<init>".equals(delegateMethod.name)) {
-            delegateInvokeType = H_NEWINVOKESPECIAL;
-        } else if (Modifier.isStatic(delegateMethod.modifiers)) {
+        if (Modifier.isStatic(delegateMethod.modifiers)) {
             delegateInvokeType = H_INVOKESTATIC;
         } else if (delegateMethod.target.isInterface()) {
             delegateInvokeType = H_INVOKEINTERFACE;
@@ -126,7 +167,8 @@ public class FunctionRef {
         this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
 
         this.interfaceMethod = interfaceMethod;
-        this.delegateMethod = delegateMethod;
+        delegateTypeParameters = delegateMethod.arguments;
+        delegateReturnType = delegateMethod.rtn;
 
         factoryDescriptor = factoryMethodType.toMethodDescriptorString();
         interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
@@ -151,13 +193,37 @@ public class FunctionRef {
         isDelegateInterface = false;
 
         this.interfaceMethod = null;
-        delegateMethod = null;
+        delegateTypeParameters = null;
+        delegateReturnType = null;
 
         factoryDescriptor = null;
         interfaceType = null;
         delegateType = null;
     }
 
+    /**
+     * Looks up {@code type} from the whitelist, and returns a matching constructor.
+     */
+    private static PainlessConstructor lookup(PainlessLookup painlessLookup, Class<?> expected, String type) {
+        // check its really a functional interface
+        // for e.g. Comparable
+        PainlessMethod method = painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod;
+        if (method == null) {
+            throw new IllegalArgumentException("Cannot convert function reference [" + type + "::new] " +
+                    "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface");
+        }
+
+        // lookup requested constructor
+        PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(painlessLookup.getJavaClassFromPainlessType(type));
+        PainlessConstructor impl = struct.constructors.get(PainlessLookupUtility.buildPainlessConstructorKey(method.arguments.size()));
+
+        if (impl == null) {
+            throw new IllegalArgumentException("Unknown reference [" + type + "::new] matching [" + expected + "]");
+        }
+
+        return impl;
+    }
+
     /**
      * Looks up {@code type::call} from the whitelist, and returns a matching method.
      */
@@ -174,27 +240,22 @@ public class FunctionRef {
         // lookup requested method
         PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(painlessLookup.getJavaClassFromPainlessType(type));
         final PainlessMethod impl;
-        // ctor ref
-        if ("new".equals(call)) {
-            impl = struct.constructors.get(PainlessLookupUtility.buildPainlessMethodKey("<init>", method.arguments.size()));
-        } else {
-            // look for a static impl first
-            PainlessMethod staticImpl =
-                    struct.staticMethods.get(PainlessLookupUtility.buildPainlessMethodKey(call, method.arguments.size()));
-            if (staticImpl == null) {
-                // otherwise a virtual impl
-                final int arity;
-                if (receiverCaptured) {
-                    // receiver captured
-                    arity = method.arguments.size();
-                } else {
-                    // receiver passed
-                    arity = method.arguments.size() - 1;
-                }
-                impl = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(call, arity));
+        // look for a static impl first
+        PainlessMethod staticImpl =
+                struct.staticMethods.get(PainlessLookupUtility.buildPainlessMethodKey(call, method.arguments.size()));
+        if (staticImpl == null) {
+            // otherwise a virtual impl
+            final int arity;
+            if (receiverCaptured) {
+                // receiver captured
+                arity = method.arguments.size();
             } else {
-                impl = staticImpl;
+                // receiver passed
+                arity = method.arguments.size() - 1;
             }
+            impl = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(call, arity));
+        } else {
+            impl = staticImpl;
         }
         if (impl == null) {
             throw new IllegalArgumentException("Unknown reference [" + type + "::" + call + "] matching " +
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
index 24dcf0ebdba..835bfb5c505 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java
@@ -24,7 +24,8 @@ import java.util.Collections;
 import java.util.Map;
 
 public final class PainlessClass {
-    public final Map<String, PainlessMethod> constructors;
+    public final Map<String, PainlessConstructor> constructors;
+
     public final Map<String, PainlessMethod> staticMethods;
     public final Map<String, PainlessMethod> methods;
 
@@ -36,13 +37,14 @@ public final class PainlessClass {
 
     public final PainlessMethod functionalMethod;
 
-    PainlessClass(Map<String, PainlessMethod> constructors,
+    PainlessClass(Map<String, PainlessConstructor> constructors,
             Map<String, PainlessMethod> staticMethods, Map<String, PainlessMethod> methods,
             Map<String, PainlessField> staticFields, Map<String, PainlessField> fields,
             Map<String, MethodHandle> getterMethodHandles, Map<String, MethodHandle> setterMethodHandles,
             PainlessMethod functionalMethod) {
 
         this.constructors = Collections.unmodifiableMap(constructors);
+
         this.staticMethods = Collections.unmodifiableMap(staticMethods);
         this.methods = Collections.unmodifiableMap(methods);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
index 2f41ed5dca8..866f711ba0f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java
@@ -24,7 +24,8 @@ import java.util.HashMap;
 import java.util.Map;
 
 final class PainlessClassBuilder {
-    final Map<String, PainlessMethod> constructors;
+    final Map<String, PainlessConstructor> constructors;
+
     final Map<String, PainlessMethod> staticMethods;
     final Map<String, PainlessMethod> methods;
 
@@ -38,6 +39,7 @@ final class PainlessClassBuilder {
 
     PainlessClassBuilder() {
         constructors = new HashMap<>();
+
         staticMethods = new HashMap<>();
         methods = new HashMap<>();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java
new file mode 100644
index 00000000000..76597c1a29d
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessConstructor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.lookup;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodType;
+import java.lang.reflect.Constructor;
+import java.util.List;
+
+public class PainlessConstructor {
+    public final Constructor<?> javaConstructor;
+    public final List<Class<?>> typeParameters;
+    public final MethodHandle methodHandle;
+    public final MethodType methodType;
+
+    PainlessConstructor(Constructor<?> javaConstructor, List<Class<?>> typeParameters, MethodHandle methodHandle, MethodType methodType) {
+        this.javaConstructor = javaConstructor;
+        this.typeParameters = typeParameters;
+        this.methodHandle = methodHandle;
+        this.methodType = methodType;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index ba48fbea734..b0a11dea58a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -40,15 +40,47 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.painless.lookup.PainlessLookupUtility.CONSTRUCTOR_NAME;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.DEF_CLASS_NAME;
+import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessConstructorKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessFieldKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.buildPainlessMethodKey;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToCanonicalTypeName;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typeToJavaType;
 import static org.elasticsearch.painless.lookup.PainlessLookupUtility.typesToCanonicalTypeNames;
 
-public class PainlessLookupBuilder {
+public final class PainlessLookupBuilder {
+
+    private static class PainlessConstructorCacheKey {
+
+        private final Class<?> targetType;
+        private final List<Class<?>> typeParameters;
+
+        private PainlessConstructorCacheKey(Class<?> targetType, List<Class<?>> typeParameters) {
+            this.targetType = targetType;
+            this.typeParameters = Collections.unmodifiableList(typeParameters);
+        }
+
+        @Override
+        public boolean equals(Object object) {
+            if (this == object) {
+                return true;
+            }
+
+            if (object == null || getClass() != object.getClass()) {
+                return false;
+            }
+
+            PainlessConstructorCacheKey that = (PainlessConstructorCacheKey)object;
+
+            return Objects.equals(targetType, that.targetType) &&
+                   Objects.equals(typeParameters, that.typeParameters);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(targetType, typeParameters);
+        }
+    }
 
     private static class PainlessMethodCacheKey {
 
@@ -120,8 +152,9 @@ public class PainlessLookupBuilder {
         }
     }
 
-    private static final Map<PainlessMethodCacheKey, PainlessMethod> painlessMethodCache = new HashMap<>();
-    private static final Map<PainlessFieldCacheKey,  PainlessField>  painlessFieldCache  = new HashMap<>();
+    private static final Map<PainlessConstructorCacheKey, PainlessConstructor> painlessConstuctorCache = new HashMap<>();
+    private static final Map<PainlessMethodCacheKey,      PainlessMethod>      painlessMethodCache     = new HashMap<>();
+    private static final Map<PainlessFieldCacheKey,       PainlessField>       painlessFieldCache      = new HashMap<>();
 
     private static final Pattern CLASS_NAME_PATTERN  = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$");
     private static final Pattern METHOD_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][_a-zA-Z0-9]*$");
@@ -343,11 +376,10 @@ public class PainlessLookupBuilder {
                     "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", nsme);
         }
 
-        String painlessMethodKey = buildPainlessMethodKey(CONSTRUCTOR_NAME, typeParametersSize);
-        PainlessMethod painlessConstructor = painlessClassBuilder.constructors.get(painlessMethodKey);
+        String painlessConstructorKey = buildPainlessConstructorKey(typeParametersSize);
+        PainlessConstructor painlessConstructor = painlessClassBuilder.constructors.get(painlessConstructorKey);
 
         if (painlessConstructor == null) {
-            org.objectweb.asm.commons.Method asmConstructor = org.objectweb.asm.commons.Method.getMethod(javaConstructor);
             MethodHandle methodHandle;
 
             try {
@@ -359,17 +391,16 @@ public class PainlessLookupBuilder {
 
             MethodType methodType = methodHandle.type();
 
-            painlessConstructor = painlessMethodCache.computeIfAbsent(
-                    new PainlessMethodCacheKey(targetClass, CONSTRUCTOR_NAME, typeParameters),
-                    key -> new PainlessMethod(CONSTRUCTOR_NAME, targetClass, null, void.class, typeParameters,
-                                              asmConstructor, javaConstructor.getModifiers(), methodHandle, methodType)
+            painlessConstructor = painlessConstuctorCache.computeIfAbsent(
+                    new PainlessConstructorCacheKey(targetClass, typeParameters),
+                    key -> new PainlessConstructor(javaConstructor, typeParameters, methodHandle, methodType)
             );
 
-            painlessClassBuilder.constructors.put(painlessMethodKey, painlessConstructor);
-        } else if (painlessConstructor.arguments.equals(typeParameters) == false){
+            painlessClassBuilder.constructors.put(painlessConstructorKey, painlessConstructor);
+        } else if (painlessConstructor.typeParameters.equals(typeParameters) == false){
             throw new IllegalArgumentException("cannot have constructors " +
                     "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "] and " +
-                    "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(painlessConstructor.arguments) + "] " +
+                    "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(painlessConstructor.typeParameters) + "] " +
                     "with the same arity and different type parameters");
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
index 86d3f876638..0a181c5f1b0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
@@ -336,6 +336,13 @@ public final class PainlessLookupUtility {
                type == String.class;
     }
 
+    /**
+     * Constructs a painless constructor key used to lookup painless constructors from a painless class.
+     */
+    public static String buildPainlessConstructorKey(int constructorArity) {
+        return CONSTRUCTOR_NAME + "/" + constructorArity;
+    }
+
     /**
      * Constructs a painless method key used to lookup painless methods from a painless class.
      */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index 7b35bc1b48e..a4f7b130045 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -77,18 +77,18 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda
             // static case
             if (captured.clazz != def.class) {
                 try {
-                    ref = new FunctionRef(locals.getPainlessLookup(), expected,
+                    ref = FunctionRef.resolveFromLookup(locals.getPainlessLookup(), expected,
                             PainlessLookupUtility.typeToCanonicalTypeName(captured.clazz), call, 1);
 
                     // check casts between the interface method and the delegate method are legal
                     for (int i = 0; i < ref.interfaceMethod.arguments.size(); ++i) {
                         Class<?> from = ref.interfaceMethod.arguments.get(i);
-                        Class<?> to = ref.delegateMethod.arguments.get(i);
+                        Class<?> to = ref.delegateTypeParameters.get(i);
                         AnalyzerCaster.getLegalCast(location, from, to, false, true);
                     }
 
                     if (ref.interfaceMethod.rtn != void.class) {
-                        AnalyzerCaster.getLegalCast(location, ref.delegateMethod.rtn, ref.interfaceMethod.rtn, false, true);
+                        AnalyzerCaster.getLegalCast(location, ref.delegateReturnType, ref.interfaceMethod.rtn, false, true);
                     }
                 } catch (IllegalArgumentException e) {
                     throw createError(e);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index d787db5d41c..23fb8a4180e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -90,7 +90,7 @@ public final class EFunctionRef extends AExpression implements ILambda {
                     }
                 } else {
                     // whitelist lookup
-                    ref = new FunctionRef(locals.getPainlessLookup(), expected, type, call, 0);
+                    ref = FunctionRef.resolveFromLookup(locals.getPainlessLookup(), expected, type, call, 0);
                 }
 
             } catch (IllegalArgumentException e) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index e0af653d209..bd4bea95ea0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -23,10 +23,12 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.Method;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -38,7 +40,7 @@ import java.util.Set;
 public final class EListInit extends AExpression {
     private final List<AExpression> values;
 
-    private PainlessMethod constructor = null;
+    private PainlessConstructor constructor = null;
     private PainlessMethod method = null;
 
     public EListInit(Location location, List<AExpression> values) {
@@ -62,8 +64,8 @@ public final class EListInit extends AExpression {
 
         actual = ArrayList.class;
 
-        constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors
-                .get(PainlessLookupUtility.buildPainlessMethodKey("<init>", 0));
+        constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors.get(
+                PainlessLookupUtility.buildPainlessConstructorKey(0));
 
         if (constructor == null) {
             throw createError(new IllegalStateException("Illegal tree structure."));
@@ -92,7 +94,8 @@ public final class EListInit extends AExpression {
 
         writer.newInstance(MethodWriter.getType(actual));
         writer.dup();
-        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
+        writer.invokeConstructor(
+                    Type.getType(constructor.javaConstructor.getDeclaringClass()), Method.getMethod(constructor.javaConstructor));
 
         for (AExpression value : values) {
             writer.dup();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index d81f08dc3cc..5bef62a4b18 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -23,10 +23,12 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.Method;
 
 import java.util.HashMap;
 import java.util.List;
@@ -39,7 +41,7 @@ public final class EMapInit extends AExpression {
     private final List<AExpression> keys;
     private final List<AExpression> values;
 
-    private PainlessMethod constructor = null;
+    private PainlessConstructor constructor = null;
     private PainlessMethod method = null;
 
     public EMapInit(Location location, List<AExpression> keys, List<AExpression> values) {
@@ -68,8 +70,8 @@ public final class EMapInit extends AExpression {
 
         actual = HashMap.class;
 
-        constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors
-                .get(PainlessLookupUtility.buildPainlessMethodKey("<init>", 0));
+        constructor = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors.get(
+                PainlessLookupUtility.buildPainlessConstructorKey(0));
 
         if (constructor == null) {
             throw createError(new IllegalStateException("Illegal tree structure."));
@@ -111,7 +113,8 @@ public final class EMapInit extends AExpression {
 
         writer.newInstance(MethodWriter.getType(actual));
         writer.dup();
-        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
+        writer.invokeConstructor(
+                    Type.getType(constructor.javaConstructor.getDeclaringClass()), Method.getMethod(constructor.javaConstructor));
 
         for (int index = 0; index < keys.size(); ++index) {
             AExpression key = keys.get(index);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
index f092a17c9fc..4e08f257386 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java
@@ -24,9 +24,10 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.objectweb.asm.Type;
+import org.objectweb.asm.commons.Method;
 
 import java.util.List;
 import java.util.Objects;
@@ -40,7 +41,7 @@ public final class ENewObj extends AExpression {
     private final String type;
     private final List<AExpression> arguments;
 
-    private PainlessMethod constructor;
+    private PainlessConstructor constructor;
 
     public ENewObj(Location location, String type, List<AExpression> arguments) {
         super(location);
@@ -65,16 +66,16 @@ public final class ENewObj extends AExpression {
         }
 
         PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual);
-        constructor = struct.constructors.get(PainlessLookupUtility.buildPainlessMethodKey("<init>", arguments.size()));
+        constructor = struct.constructors.get(PainlessLookupUtility.buildPainlessConstructorKey(arguments.size()));
 
         if (constructor != null) {
-            Class<?>[] types = new Class<?>[constructor.arguments.size()];
-            constructor.arguments.toArray(types);
+            Class<?>[] types = new Class<?>[constructor.typeParameters.size()];
+            constructor.typeParameters.toArray(types);
 
-            if (constructor.arguments.size() != arguments.size()) {
+            if (constructor.typeParameters.size() != arguments.size()) {
                 throw createError(new IllegalArgumentException(
                         "When calling constructor on type [" + PainlessLookupUtility.typeToCanonicalTypeName(actual) + "] " +
-                        "expected [" + constructor.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
+                        "expected [" + constructor.typeParameters.size() + "] arguments, but found [" + arguments.size() + "]."));
             }
 
             for (int argument = 0; argument < arguments.size(); ++argument) {
@@ -107,7 +108,8 @@ public final class ENewObj extends AExpression {
             argument.write(writer, globals);
         }
 
-        writer.invokeConstructor(Type.getType(constructor.target), constructor.method);
+        writer.invokeConstructor(
+                    Type.getType(constructor.javaConstructor.getDeclaringClass()), Method.getMethod(constructor.javaConstructor));
     }
 
     @Override
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index ff0d4231175..65429154301 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.painless.lookup.PainlessClass;
+import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessField;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
@@ -57,7 +58,8 @@ public class PainlessDocGenerator {
     private static final Logger logger = ESLoggerFactory.getLogger(PainlessDocGenerator.class);
     private static final Comparator<PainlessField> FIELD_NAME = comparing(f -> f.name);
     private static final Comparator<PainlessMethod> METHOD_NAME = comparing(m -> m.name);
-    private static final Comparator<PainlessMethod> NUMBER_OF_ARGS = comparing(m -> m.arguments.size());
+    private static final Comparator<PainlessMethod> METHOD_NUMBER_OF_PARAMS = comparing(m -> m.arguments.size());
+    private static final Comparator<PainlessConstructor> CONSTRUCTOR_NUMBER_OF_PARAMS = comparing(m -> m.typeParameters.size());
 
     public static void main(String[] args) throws IOException {
         Path apiRootPath = PathUtils.get(args[0]);
@@ -103,12 +105,15 @@ public class PainlessDocGenerator {
 
                     Consumer<PainlessField> documentField = field -> PainlessDocGenerator.documentField(typeStream, field);
                     Consumer<PainlessMethod> documentMethod = method -> PainlessDocGenerator.documentMethod(typeStream, method);
+                    Consumer<PainlessConstructor> documentConstructor =
+                            constructor -> PainlessDocGenerator.documentConstructor(typeStream, constructor);
                     struct.staticFields.values().stream().sorted(FIELD_NAME).forEach(documentField);
                     struct.fields.values().stream().sorted(FIELD_NAME).forEach(documentField);
-                    struct.staticMethods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(documentMethod);
-                    struct.constructors.values().stream().sorted(NUMBER_OF_ARGS).forEach(documentMethod);
+                    struct.staticMethods.values().stream().sorted(
+                            METHOD_NAME.thenComparing(METHOD_NUMBER_OF_PARAMS)).forEach(documentMethod);
+                    struct.constructors.values().stream().sorted(CONSTRUCTOR_NUMBER_OF_PARAMS).forEach(documentConstructor);
                     Map<String, Class<?>> inherited = new TreeMap<>();
-                    struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(method -> {
+                    struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(METHOD_NUMBER_OF_PARAMS)).forEach(method -> {
                         if (method.target == clazz) {
                             documentMethod(typeStream, method);
                         } else {
@@ -164,6 +169,41 @@ public class PainlessDocGenerator {
         stream.println();
     }
 
+    /**
+     * Document a constructor.
+     */
+    private static void documentConstructor(PrintStream stream, PainlessConstructor constructor) {
+        stream.print("* ++[[");
+        emitAnchor(stream, constructor);
+        stream.print("]]");
+
+        String javadocRoot = javadocRoot(constructor.javaConstructor.getDeclaringClass());
+        emitJavadocLink(stream, javadocRoot, constructor);
+        stream.print('[');
+
+        stream.print(constructorName(constructor));
+
+        stream.print("](");
+        boolean first = true;
+        for (Class<?> arg : constructor.typeParameters) {
+            if (first) {
+                first = false;
+            } else {
+                stream.print(", ");
+            }
+            emitType(stream, arg);
+        }
+        stream.print(")++");
+
+        if (javadocRoot.equals("java8")) {
+            stream.print(" (");
+            emitJavadocLink(stream, "java9", constructor);
+            stream.print("[java 9])");
+        }
+
+        stream.println();
+    }
+
     /**
      * Document a method.
      */
@@ -176,10 +216,8 @@ public class PainlessDocGenerator {
             stream.print("static ");
         }
 
-        if (false == method.name.equals("<init>")) {
-            emitType(stream, method.rtn);
-            stream.print(' ');
-        }
+        emitType(stream, method.rtn);
+        stream.print(' ');
 
         String javadocRoot = javadocRoot(method);
         emitJavadocLink(stream, javadocRoot, method);
@@ -216,6 +254,17 @@ public class PainlessDocGenerator {
         stream.print(PainlessLookupUtility.typeToCanonicalTypeName(clazz).replace('.', '-'));
     }
 
+    /**
+     * Anchor text for a {@link PainlessConstructor}.
+     */
+    private static void emitAnchor(PrintStream stream, PainlessConstructor constructor) {
+        emitAnchor(stream, constructor.javaConstructor.getDeclaringClass());
+        stream.print('-');
+        stream.print(constructorName(constructor));
+        stream.print('-');
+        stream.print(constructor.typeParameters.size());
+    }
+
     /**
      * Anchor text for a {@link PainlessMethod}.
      */
@@ -236,8 +285,12 @@ public class PainlessDocGenerator {
         stream.print(field.name);
     }
 
+    private static String constructorName(PainlessConstructor constructor) {
+        return PainlessLookupUtility.typeToCanonicalTypeName(constructor.javaConstructor.getDeclaringClass());
+    }
+
     private static String methodName(PainlessMethod method) {
-        return method.name.equals("<init>") ? PainlessLookupUtility.typeToCanonicalTypeName(method.target) : method.name;
+        return PainlessLookupUtility.typeToCanonicalTypeName(method.target);
     }
 
     /**
@@ -269,6 +322,34 @@ public class PainlessDocGenerator {
         }
     }
 
+    /**
+     * Emit an external link to Javadoc for a {@link PainlessMethod}.
+     *
+     * @param root name of the root uri variable
+     */
+    private static void emitJavadocLink(PrintStream stream, String root, PainlessConstructor constructor) {
+        stream.print("link:{");
+        stream.print(root);
+        stream.print("-javadoc}/");
+        stream.print(classUrlPath(constructor.javaConstructor.getDeclaringClass()));
+        stream.print(".html#");
+        stream.print(constructorName(constructor));
+        stream.print("%2D");
+        boolean first = true;
+        for (Class<?> clazz: constructor.typeParameters) {
+            if (first) {
+                first = false;
+            } else {
+                stream.print("%2D");
+            }
+            stream.print(clazz.getName());
+            if (clazz.isArray()) {
+                stream.print(":A");
+            }
+        }
+        stream.print("%2D");
+    }
+
     /**
      * Emit an external link to Javadoc for a {@link PainlessMethod}.
      *

From 670630948b4831704fbaf4821554d4ca52a5a42d Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 30 Jul 2018 18:07:49 -0400
Subject: [PATCH 229/260] Switch security spi example to new style Requests
 (#32341)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `x-pack/qa/security-example-spi-extension`
project to use the new versions.
---
 .../example/realm/CustomRealmIT.java          | 14 +++++----
 .../example/role/CustomRolesProviderIT.java   | 29 ++++++++++++-------
 2 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java
index 65ec595a0d4..f7bdb0d0baa 100644
--- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java
+++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.example.realm;
 
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
@@ -50,7 +51,7 @@ public class CustomRealmIT extends ESIntegTestCase {
 
     public void testHttpConnectionWithNoAuthentication() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/");
+            getRestClient().performRequest(new Request("GET", "/"));
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();
@@ -61,9 +62,12 @@ public class CustomRealmIT extends ESIntegTestCase {
     }
 
     public void testHttpAuthentication() throws Exception {
-        Response response = getRestClient().performRequest("GET", "/",
-                new BasicHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER),
-                new BasicHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString()));
+        Request request = new Request("GET", "/");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER);
+        options.addHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString());
+        request.setOptions(options);
+        Response response = getRestClient().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
index bd3c53a3b41..57a895848e3 100644
--- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
+++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.example.role;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
@@ -33,10 +34,17 @@ import static org.hamcrest.Matchers.is;
  * Integration test for custom roles providers.
  */
 public class CustomRolesProviderIT extends ESIntegTestCase {
-
     private static final String TEST_USER = "test_user";
     private static final String TEST_PWD = "change_me";
 
+    private static final RequestOptions AUTH_OPTIONS;
+    static {
+        RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+        options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue(TEST_USER, new SecureString(TEST_PWD.toCharArray())));
+        AUTH_OPTIONS = options.build();
+    }
+
     @Override
     protected Settings externalClusterClientSettings() {
         return Settings.builder()
@@ -59,7 +67,9 @@ public class CustomRolesProviderIT extends ESIntegTestCase {
     public void testAuthorizedCustomRoleSucceeds() throws Exception {
         setupTestUser(ROLE_B);
         // roleB has all permissions on index "foo", so creating "foo" should succeed
-        Response response = getRestClient().performRequest("PUT", "/" + INDEX, authHeader());
+        Request request = new Request("PUT", "/" + INDEX);
+        request.setOptions(AUTH_OPTIONS);
+        Response response = getRestClient().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
@@ -71,7 +81,9 @@ public class CustomRolesProviderIT extends ESIntegTestCase {
         setupTestUser(ROLE_A);
         // roleB has all permissions on index "foo", so creating "foo" should succeed
         try {
-            getRestClient().performRequest("PUT", "/" + INDEX, authHeader());
+            Request request = new Request("PUT", "/" + INDEX);
+            request.setOptions(AUTH_OPTIONS);
+            getRestClient().performRequest(request);
             fail(ROLE_A + " should not be authorized to create index " + INDEX);
         } catch (ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
@@ -82,15 +94,12 @@ public class CustomRolesProviderIT extends ESIntegTestCase {
         setupTestUser("unknown");
         // roleB has all permissions on index "foo", so creating "foo" should succeed
         try {
-            getRestClient().performRequest("PUT", "/" + INDEX, authHeader());
+            Request request = new Request("PUT", "/" + INDEX);
+            request.setOptions(AUTH_OPTIONS);
+            getRestClient().performRequest(request);
             fail(ROLE_A + " should not be authorized to create index " + INDEX);
         } catch (ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
         }
     }
-
-    private BasicHeader authHeader() {
-        return new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                               basicAuthHeaderValue(TEST_USER, new SecureString(TEST_PWD.toCharArray())));
-    }
 }

From 4101fc4e3d0bd3ccc64c99e1bdd93dee6d8df4d8 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 30 Jul 2018 18:16:26 -0400
Subject: [PATCH 230/260] Switch security to new style Requests (#32290)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `x-pack/plugin/security` project to use the new
versions.
---
 .../integration/BulkUpdateTests.java          | 63 +++++++++----------
 .../integration/ClearRealmsCacheTests.java    | 16 +++--
 .../integration/IndexPrivilegeTests.java      | 12 ++--
 .../elasticsearch/license/LicensingTests.java | 32 ++++++----
 .../test/NativeRealmIntegTestCase.java        | 36 +++++------
 .../xpack/security/SecurityPluginTests.java   | 17 ++---
 .../security/audit/index/AuditTrailTests.java | 23 ++++---
 .../xpack/security/authc/RunAsIntegTests.java | 53 ++++++++--------
 .../authc/pki/PkiOptionalClientAuthTests.java | 15 +++--
 .../action/RestAuthenticateActionTests.java   | 16 ++---
 .../user/AnonymousUserIntegTests.java         |  3 +-
 .../xpack/ssl/SSLClientAuthTests.java         | 12 ++--
 12 files changed, 161 insertions(+), 137 deletions(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java
index 03ace7fb71d..bb0036e9f87 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java
@@ -5,15 +5,12 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.Header;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.client.Response;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -24,10 +21,8 @@ import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 
 import java.io.IOException;
-import java.util.Collections;
 
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
 
 public class BulkUpdateTests extends SecurityIntegTestCase {
 
@@ -77,46 +72,48 @@ public class BulkUpdateTests extends SecurityIntegTestCase {
 
     public void testThatBulkUpdateDoesNotLoseFieldsHttp() throws IOException {
         final String path = "/index1/type/1";
-        final Header basicAuthHeader = new BasicHeader("Authorization",
-                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        final RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder();
+        optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        final RequestOptions options = optionsBuilder.build();
 
-        StringEntity body = new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON);
-        Response response = getRestClient().performRequest("PUT", path, Collections.emptyMap(), body, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(201));
+        Request createRequest = new Request("PUT", path);
+        createRequest.setOptions(options);
+        createRequest.setJsonEntity("{\"test\":\"test\"}");
+        getRestClient().performRequest(createRequest);
 
-        response = getRestClient().performRequest("GET", path, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-        assertThat(EntityUtils.toString(response.getEntity()), containsString("\"test\":\"test\""));
+        Request getRequest = new Request("GET", path);
+        getRequest.setOptions(options);
+        assertThat(EntityUtils.toString(getRestClient().performRequest(getRequest).getEntity()), containsString("\"test\":\"test\""));
 
         if (randomBoolean()) {
             flushAndRefresh();
         }
 
         //update with new field
-        body = new StringEntity("{\"doc\": {\"not test\": \"not test\"}}", ContentType.APPLICATION_JSON);
-        response = getRestClient().performRequest("POST", path + "/_update", Collections.emptyMap(), body, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+        Request updateRequest = new Request("POST", path + "/_update");
+        updateRequest.setOptions(options);
+        updateRequest.setJsonEntity("{\"doc\": {\"not test\": \"not test\"}}");
+        getRestClient().performRequest(updateRequest);
 
-        response = getRestClient().performRequest("GET", path, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-        String responseBody = EntityUtils.toString(response.getEntity());
-        assertThat(responseBody, containsString("\"test\":\"test\""));
-        assertThat(responseBody, containsString("\"not test\":\"not test\""));
+        String afterUpdate = EntityUtils.toString(getRestClient().performRequest(getRequest).getEntity());
+        assertThat(afterUpdate, containsString("\"test\":\"test\""));
+        assertThat(afterUpdate, containsString("\"not test\":\"not test\""));
 
         // this part is important. Without this, the document may be read from the translog which would bypass the bug where
         // FLS kicks in because the request can't be found and only returns meta fields
         flushAndRefresh();
 
-        body = new StringEntity("{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n" +
-                "{\"doc\": {\"bulk updated\":\"bulk updated\"}}\n", ContentType.APPLICATION_JSON);
-        response = getRestClient().performRequest("POST", "/_bulk", Collections.emptyMap(), body, basicAuthHeader);
-        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+        Request bulkRequest = new Request("POST", "/_bulk");
+        bulkRequest.setOptions(options);
+        bulkRequest.setJsonEntity(
+                "{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n" +
+                "{\"doc\": {\"bulk updated\":\"bulk updated\"}}\n");
+        getRestClient().performRequest(bulkRequest);
 
-        response = getRestClient().performRequest("GET", path, basicAuthHeader);
-        responseBody = EntityUtils.toString(response.getEntity());
-        assertThat(responseBody, containsString("\"test\":\"test\""));
-        assertThat(responseBody, containsString("\"not test\":\"not test\""));
-        assertThat(responseBody, containsString("\"bulk updated\":\"bulk updated\""));
+        String afterBulk = EntityUtils.toString(getRestClient().performRequest(getRequest).getEntity());
+        assertThat(afterBulk, containsString("\"test\":\"test\""));
+        assertThat(afterBulk, containsString("\"not test\":\"not test\""));
+        assertThat(afterBulk, containsString("\"bulk updated\":\"bulk updated\""));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
index 8b9e1954264..fc02a5c4d62 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
@@ -160,10 +161,15 @@ public class ClearRealmsCacheTests extends SecurityIntegTestCase {
         }
 
         static void executeHttpRequest(String path, Map<String, String> params) throws Exception {
-            Response response = getRestClient().performRequest("POST", path, params,
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                                    new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))));
+            Request request = new Request("POST", path);
+            for (Map.Entry<String, String> param : params.entrySet()) {
+                request.addParameter(param.getKey(), param.getValue());
+            }
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                    new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+            request.setOptions(options);
+            Response response = getRestClient().performRequest(request);
             assertNotNull(response.getEntity());
             assertTrue(EntityUtils.toString(response.getEntity()).contains("cluster_name"));
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
index 825bfcd432f..57262822982 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
@@ -388,9 +389,12 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
 
     public void testThatUnknownUserIsRejectedProperly() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue("idonotexist", new SecureString("passwd".toCharArray()))));
+            Request request = new Request("GET", "/");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization",
+                    UsernamePasswordToken.basicAuthHeaderValue("idonotexist", new SecureString("passwd".toCharArray())));
+            request.setOptions(options);
+            getRestClient().performRequest(request);
             fail("request should have failed");
         } catch(ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
index 012050f4259..7a35b0bc422 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.license;
 
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
@@ -15,6 +14,8 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
@@ -189,31 +190,36 @@ public class LicensingTests extends SecurityIntegTestCase {
     }
 
     public void testRestAuthenticationByLicenseType() throws Exception {
-        Response response = getRestClient().performRequest("GET", "/");
+        Response unauthorizedRootResponse = getRestClient().performRequest(new Request("GET", "/"));
         // the default of the licensing tests is basic
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        assertThat(unauthorizedRootResponse.getStatusLine().getStatusCode(), is(200));
         ResponseException e = expectThrows(ResponseException.class,
-                () -> getRestClient().performRequest("GET", "/_xpack/security/_authenticate"));
+                () -> getRestClient().performRequest(new Request("GET", "/_xpack/security/_authenticate")));
         assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
 
         // generate a new license with a mode that enables auth
         License.OperationMode mode = randomFrom(License.OperationMode.GOLD, License.OperationMode.TRIAL,
                 License.OperationMode.PLATINUM, License.OperationMode.STANDARD);
         enableLicensing(mode);
-        e = expectThrows(ResponseException.class, () -> getRestClient().performRequest("GET", "/"));
+        e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(new Request("GET", "/")));
         assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
         e = expectThrows(ResponseException.class,
-            () -> getRestClient().performRequest("GET", "/_xpack/security/_authenticate"));
+            () -> getRestClient().performRequest(new Request("GET", "/_xpack/security/_authenticate")));
         assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
 
-        final String basicAuthValue = UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()));
-        response = getRestClient().performRequest("GET", "/", new BasicHeader("Authorization", basicAuthValue));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        response = getRestClient().performRequest("GET", "/_xpack/security/_authenticate",
-                    new BasicHeader("Authorization", basicAuthValue));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder();
+        optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        RequestOptions options = optionsBuilder.build();
 
+        Request rootRequest = new Request("GET", "/");
+        rootRequest.setOptions(options);
+        Response authorizedRootResponse = getRestClient().performRequest(rootRequest);
+        assertThat(authorizedRootResponse.getStatusLine().getStatusCode(), is(200));
+        Request authenticateRequest = new Request("GET", "/_xpack/security/_authenticate");
+        authenticateRequest.setOptions(options);
+        Response authorizedAuthenticateResponse = getRestClient().performRequest(authenticateRequest);
+        assertThat(authorizedAuthenticateResponse.getStatusLine().getStatusCode(), is(200));
     }
 
     public void testSecurityActionsByLicenseType() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
index 72550832779..af5b73d889d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
@@ -5,12 +5,8 @@
  */
 package org.elasticsearch.test;
 
-
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.nio.entity.NStringEntity;
-import org.elasticsearch.client.Response;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.set.Sets;
@@ -26,7 +22,6 @@ import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Set;
 
 /**
@@ -82,23 +77,22 @@ public abstract class NativeRealmIntegTestCase extends SecurityIntegTestCase {
     public void setupReservedPasswords(RestClient restClient) throws IOException {
         logger.info("setting up reserved passwords for test");
         {
-            String payload = "{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}";
-            HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON);
-            BasicHeader authHeader = new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, BOOTSTRAP_PASSWORD));
-            String route = "/_xpack/security/user/elastic/_password";
-            Response response = restClient.performRequest("PUT", route, Collections.emptyMap(), entity, authHeader);
-            assertEquals(response.getStatusLine().getReasonPhrase(), 200, response.getStatusLine().getStatusCode());
+            Request request = new Request("PUT", "/_xpack/security/user/elastic/_password");
+            request.setJsonEntity("{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, BOOTSTRAP_PASSWORD));
+            request.setOptions(options);
+            restClient.performRequest(request);
         }
 
+        RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder();
+        optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, reservedPassword));
+        RequestOptions options = optionsBuilder.build();
         for (String username : Arrays.asList(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME)) {
-            String payload = "{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}";
-            HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON);
-            BasicHeader authHeader = new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    UsernamePasswordToken.basicAuthHeaderValue(ElasticUser.NAME, reservedPassword));
-            String route = "/_xpack/security/user/" + username + "/_password";
-            Response response = restClient.performRequest("PUT", route, Collections.emptyMap(), entity, authHeader);
-            assertEquals(response.getStatusLine().getReasonPhrase(), 200, response.getStatusLine().getStatusCode());
+            Request request = new Request("PUT", "/_xpack/security/user/" + username + "/_password");
+            request.setJsonEntity("{\"password\": \"" + new String(reservedPassword.getChars()) + "\"}");
+            request.setOptions(options);
+            restClient.performRequest(request);
         }
         logger.info("setting up reserved passwords finished");
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java
index 368449adeed..d57a5d151a9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java
@@ -5,14 +5,14 @@
  */
 package org.elasticsearch.xpack.security;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
-import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 
 import java.io.IOException;
 
@@ -31,17 +31,20 @@ public class SecurityPluginTests extends SecurityIntegTestCase {
     public void testThatPluginIsLoaded() throws IOException {
         try {
             logger.info("executing unauthorized request to /_xpack info");
-            getRestClient().performRequest("GET", "/_xpack");
+            getRestClient().performRequest(new Request("GET", "/_xpack"));
             fail("request should have failed");
         } catch(ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(UNAUTHORIZED.getStatus()));
         }
 
         logger.info("executing authorized request to /_xpack infos");
-        Response response = getRestClient().performRequest("GET", "/_xpack",
-                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))));
+
+        Request request = new Request("GET", "/_xpack");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        request.setOptions(options);
+        Response response = getRestClient().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), is(OK.getStatus()));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
index 453820ea519..ef3c6aa56ae 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/index/AuditTrailTests.java
@@ -5,12 +5,13 @@
  */
 package org.elasticsearch.xpack.security.audit.index;
 
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
@@ -89,10 +90,12 @@ public class AuditTrailTests extends SecurityIntegTestCase {
 
     public void testAuditAccessDeniedWithRunAsUser() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/.security/_search",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(AUTHENTICATE_USER, TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, EXECUTE_USER));
+            Request request = new Request("GET", "/.security/_search");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(AUTHENTICATE_USER, TEST_PASSWORD_SECURE_STRING));
+            options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, EXECUTE_USER);
+            request.setOptions(options);
+            getRestClient().performRequest(request);
             fail("request should have failed");
         } catch (final ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
@@ -111,10 +114,12 @@ public class AuditTrailTests extends SecurityIntegTestCase {
 
     public void testAuditRunAsDeniedEmptyUser() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/.security/_search",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(AUTHENTICATE_USER, TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, ""));
+            Request request = new Request("GET", "/.security/_search");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(AUTHENTICATE_USER, TEST_PASSWORD_SECURE_STRING));
+            options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, "");
+            request.setOptions(options);
+            getRestClient().performRequest(request);
             fail("request should have failed");
         } catch (final ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
index 9311b9c02f6..6d5c6770bf2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.security.authc;
 
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.client.Response;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.SecureString;
@@ -126,11 +126,13 @@ public class RunAsIntegTests extends SecurityIntegTestCase {
     public void testUserImpersonationUsingHttp() throws Exception {
         // use the transport client user and try to run as
         try {
-            getRestClient().performRequest("GET", "/_nodes",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER,
-                                    TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, SecuritySettingsSource.TEST_USER_NAME));
+            Request request = new Request("GET", "/_nodes");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization",
+                    UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER, TEST_PASSWORD_SECURE_STRING));
+            options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, SecuritySettingsSource.TEST_USER_NAME);
+            request.setOptions(options);
+            getRestClient().performRequest(request);
             fail("request should have failed");
         } catch(ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
@@ -139,10 +141,11 @@ public class RunAsIntegTests extends SecurityIntegTestCase {
         if (runAsHasSuperUserRole == false) {
             try {
                 //the run as user shouldn't have access to the nodes api
-                getRestClient().performRequest("GET", "/_nodes",
-                        new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                                UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                                        TEST_PASSWORD_SECURE_STRING)));
+                Request request = new Request("GET", "/_nodes");
+                RequestOptions.Builder options = request.getOptions().toBuilder();
+                options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, TEST_PASSWORD_SECURE_STRING));
+                request.setOptions(options);
+                getRestClient().performRequest(request);
                 fail("request should have failed");
             } catch (ResponseException e) {
                 assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
@@ -150,12 +153,7 @@ public class RunAsIntegTests extends SecurityIntegTestCase {
         }
 
         // but when running as a different user it should work
-        Response response = getRestClient().performRequest("GET", "/_nodes",
-                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                                TEST_PASSWORD_SECURE_STRING)),
-                new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, SecuritySettingsSource.TEST_USER_NAME));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        getRestClient().performRequest(requestForUserRunAsUser(SecuritySettingsSource.TEST_USER_NAME));
     }
 
     public void testEmptyUserImpersonationHeader() throws Exception {
@@ -183,11 +181,7 @@ public class RunAsIntegTests extends SecurityIntegTestCase {
 
     public void testEmptyHeaderUsingHttp() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/_nodes",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                                    TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, ""));
+            getRestClient().performRequest(requestForUserRunAsUser(""));
             fail("request should have failed");
         } catch(ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
@@ -219,17 +213,22 @@ public class RunAsIntegTests extends SecurityIntegTestCase {
 
     public void testNonExistentRunAsUserUsingHttp() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/_nodes",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                                    TEST_PASSWORD_SECURE_STRING)),
-                    new BasicHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, "idontexist"));
+            getRestClient().performRequest(requestForUserRunAsUser("idontexist"));
             fail("request should have failed");
         } catch (ResponseException e) {
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
         }
     }
 
+    private static Request requestForUserRunAsUser(String user) {
+        Request request = new Request("GET", "/_nodes");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, TEST_PASSWORD_SECURE_STRING));
+        options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, user);
+        request.setOptions(options);
+        return request;
+    }
+
     // build our own here to better mimic an actual client...
     TransportClient getTransportClient(Settings extraSettings) {
         NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java
index d47ffe4a344..4fb94c74949 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java
@@ -5,8 +5,9 @@
  */
 package org.elasticsearch.xpack.security.authc.pki;
 
-import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -76,13 +77,15 @@ public class PkiOptionalClientAuthTests extends SecuritySingleNodeTestCase {
     public void testRestClientWithoutClientCertificate() throws Exception {
         SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(getSSLContext());
         try (RestClient restClient = createRestClient(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy), "https")) {
-            ResponseException e = expectThrows(ResponseException.class, () -> restClient.performRequest("GET", "_nodes"));
+            ResponseException e = expectThrows(ResponseException.class, () -> restClient.performRequest(new Request("GET", "_nodes")));
             assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
 
-            Response response = restClient.performRequest("GET", "_nodes",
-                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                                    new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))));
+            Request request = new Request("GET", "_nodes");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                    new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+            request.setOptions(options);
+            Response response = restClient.performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), is(200));
         }
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
index 67bfc2ecdcb..13a124e4bdc 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.security.rest.action;
 
-import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
@@ -52,11 +53,12 @@ public class RestAuthenticateActionTests extends SecurityIntegTestCase {
     }
 
     public void testAuthenticateApi() throws Exception {
-        Response response = getRestClient().performRequest("GET", "/_xpack/security/_authenticate",
-                new BasicHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
-        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        Request request = new Request("GET", "/_xpack/security/_authenticate");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+        request.setOptions(options);
+        ObjectPath objectPath = ObjectPath.createFromResponse(getRestClient().performRequest(request));
         assertThat(objectPath.evaluate("username").toString(), equalTo(SecuritySettingsSource.TEST_USER_NAME));
         List<String> roles = objectPath.evaluate("roles");
         assertThat(roles.size(), is(1));
@@ -65,7 +67,7 @@ public class RestAuthenticateActionTests extends SecurityIntegTestCase {
 
     public void testAuthenticateApiWithoutAuthentication() throws Exception {
         try {
-            Response response = getRestClient().performRequest("GET", "/_xpack/security/_authenticate");
+            Response response = getRestClient().performRequest(new Request("GET", "/_xpack/security/_authenticate"));
             if (anonymousEnabled) {
                 assertThat(response.getStatusLine().getStatusCode(), is(200));
                 ObjectPath objectPath = ObjectPath.createFromResponse(response);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
index 9529a12a30a..431b3e855c6 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security.user;
 
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
@@ -46,7 +47,7 @@ public class AnonymousUserIntegTests extends SecurityIntegTestCase {
 
     public void testAnonymousViaHttp() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/_nodes");
+            getRestClient().performRequest(new Request("GET", "/_nodes"));
             fail("request should have failed");
         } catch(ResponseException e) {
             int statusCode = e.getResponse().getStatusLine().getStatusCode();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
index d205c7cd933..21da604374f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
@@ -6,12 +6,13 @@
 package org.elasticsearch.xpack.ssl;
 
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.transport.TransportClient;
@@ -71,7 +72,7 @@ public class SSLClientAuthTests extends SecurityIntegTestCase {
     public void testThatHttpFailsWithoutSslClientAuth() throws IOException {
         SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(SSLContexts.createDefault(), NoopHostnameVerifier.INSTANCE);
         try (RestClient restClient = createRestClient(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy), "https")) {
-            restClient.performRequest("GET", "/");
+            restClient.performRequest(new Request("GET", "/"));
             fail("Expected SSLHandshakeException");
         } catch (IOException e) {
             Throwable t = ExceptionsHelper.unwrap(e, CertPathBuilderException.class);
@@ -87,8 +88,11 @@ public class SSLClientAuthTests extends SecurityIntegTestCase {
     public void testThatHttpWorksWithSslClientAuth() throws IOException {
         SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(getSSLContext(), NoopHostnameVerifier.INSTANCE);
         try (RestClient restClient = createRestClient(httpClientBuilder -> httpClientBuilder.setSSLStrategy(sessionStrategy), "https")) {
-            Response response = restClient.performRequest("GET", "/",
-                    new BasicHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword())));
+            Request request = new Request("GET", "/");
+            RequestOptions.Builder options = request.getOptions().toBuilder();
+            options.addHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword()));
+            request.setOptions(options);
+            Response response = restClient.performRequest(request);
             assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
             assertThat(EntityUtils.toString(response.getEntity()), containsString("You Know, for Search"));
         }

From d75efbcf6839a63e05ae3b88c84b6324921f25b3 Mon Sep 17 00:00:00 2001
From: Tim Vernum <tim@adjective.org>
Date: Tue, 31 Jul 2018 09:07:47 +1000
Subject: [PATCH 231/260] Make get all app privs requires "*" permission
 (#32460)

The default behaviour for "GetPrivileges" is to get all application
privileges. This should only be allowed if the user has access to
the "*" application.
---
 .../action/privilege/GetPrivilegesRequest.java     |  2 +-
 .../privilege/ConditionalClusterPrivileges.java    |  4 +++-
 .../ManageApplicationPrivilegesTests.java          | 14 ++++++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
index 559e0ab8d98..585fb372d32 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
@@ -50,7 +50,7 @@ public final class GetPrivilegesRequest extends ActionRequest implements Applica
 
     @Override
     public Collection<String> getApplicationNames() {
-        return Collections.singleton(application);
+        return application == null ? Collections.emptySet() : Collections.singleton(application);
     }
 
     public void privileges(String... privileges) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
index c068c77781b..e204d89b1c0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConditionalClusterPrivileges.java
@@ -135,7 +135,9 @@ public final class ConditionalClusterPrivileges {
             this.requestPredicate = request -> {
                 if (request instanceof ApplicationPrivilegesRequest) {
                     final ApplicationPrivilegesRequest privRequest = (ApplicationPrivilegesRequest) request;
-                    return privRequest.getApplicationNames().stream().allMatch(application -> applicationPredicate.test(application));
+                    final Collection<String> requestApplicationNames = privRequest.getApplicationNames();
+                    return requestApplicationNames.isEmpty() ? this.applicationNames.contains("*")
+                        : requestApplicationNames.stream().allMatch(application -> applicationPredicate.test(application));
                 }
                 return false;
             };
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
index a5c1bbc98d1..9c113d4ff0f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageApplicationPrivilegesTests.java
@@ -50,6 +50,7 @@ import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class ManageApplicationPrivilegesTests extends ESTestCase {
 
@@ -140,6 +141,19 @@ public class ManageApplicationPrivilegesTests extends ESTestCase {
         assertThat(cloudAndSwiftypePredicate, not(predicateMatches(putKibana)));
     }
 
+    public void testSecurityForGetAllApplicationPrivileges() {
+        final GetPrivilegesRequest getAll = new GetPrivilegesRequest();
+        getAll.application(null);
+        getAll.privileges(new String[0]);
+
+        assertThat(getAll.validate(), nullValue());
+
+        final ManageApplicationPrivileges kibanaOnly = new ManageApplicationPrivileges(Sets.newHashSet("kibana-*"));
+        final ManageApplicationPrivileges allApps = new ManageApplicationPrivileges(Sets.newHashSet("*"));
+
+        assertThat(kibanaOnly.getRequestPredicate(), not(predicateMatches(getAll)));
+        assertThat(allApps.getRequestPredicate(), predicateMatches(getAll));
+    }
 
     private ManageApplicationPrivileges clone(ManageApplicationPrivileges original) {
         return new ManageApplicationPrivileges(new LinkedHashSet<>(original.getApplicationNames()));

From f0b36679eca9653c3358842b137fdf354a328901 Mon Sep 17 00:00:00 2001
From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com>
Date: Tue, 31 Jul 2018 10:59:36 +1000
Subject: [PATCH 232/260] [Kerberos] Remove Kerberos bootstrap checks (#32451)

This commit removes Kerberos bootstrap checks as they were more
validation checks and better done in Kerberos realm constructor
than as bootstrap checks. This also moves the check
for one Kerberos realm per node to where we initialize realms.
This commit adds few validations which were missing earlier
like missing read permissions on keytab file or if it is directory
to throw exception with error message.
---
 .../xpack/security/Security.java              |   4 +-
 .../xpack/security/authc/Realms.java          |   9 ++
 .../authc/kerberos/KerberosRealm.java         |  11 ++
 .../kerberos/KerberosRealmBootstrapCheck.java |  69 -----------
 .../xpack/security/authc/RealmsTests.java     |  44 +++++--
 .../KerberosRealmBootstrapCheckTests.java     | 114 ------------------
 .../authc/kerberos/KerberosRealmTests.java    |  54 +++++++++
 7 files changed, 106 insertions(+), 199 deletions(-)
 delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java
 delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index f4bb4b7eb3b..df5e8dcae1b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -176,7 +176,6 @@ import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
-import org.elasticsearch.xpack.security.authc.kerberos.KerberosRealmBootstrapCheck;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener;
@@ -306,8 +305,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 new PasswordHashingAlgorithmBootstrapCheck(),
                 new FIPS140SecureSettingsBootstrapCheck(settings, env),
                 new FIPS140JKSKeystoreBootstrapCheck(settings),
-                new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings),
-                new KerberosRealmBootstrapCheck(env)));
+                new FIPS140PasswordHashingAlgorithmBootstrapCheck(settings)));
             checks.addAll(InternalRealms.getBootstrapChecks(settings, env));
             this.bootstrapChecks = Collections.unmodifiableList(checks);
             Automatons.updateMaxDeterminizedStates(settings);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
index 0284ae9a05f..8b80c1f1d1c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
@@ -35,6 +35,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 
 
 /**
@@ -152,6 +153,7 @@ public class Realms extends AbstractComponent implements Iterable<Realm> {
         Settings realmsSettings = RealmSettings.get(settings);
         Set<String> internalTypes = new HashSet<>();
         List<Realm> realms = new ArrayList<>();
+        List<String> kerberosRealmNames = new ArrayList<>();
         for (String name : realmsSettings.names()) {
             Settings realmSettings = realmsSettings.getAsSettings(name);
             String type = realmSettings.get("type");
@@ -178,6 +180,13 @@ public class Realms extends AbstractComponent implements Iterable<Realm> {
                 }
                 internalTypes.add(type);
             }
+            if (KerberosRealmSettings.TYPE.equals(type)) {
+                kerberosRealmNames.add(name);
+                if (kerberosRealmNames.size() > 1) {
+                    throw new IllegalArgumentException("multiple realms " + kerberosRealmNames.toString() + " configured of type [" + type
+                            + "], [" + type + "] can only have one such realm configured");
+                }
+            }
             realms.add(factory.create(config));
         }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
index b4a8b6aabf0..71eeb8b2398 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
@@ -25,6 +25,7 @@ import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.ietf.jgss.GSSException;
 
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.List;
@@ -87,6 +88,16 @@ public final class KerberosRealm extends Realm implements CachingRealm {
         this.kerberosTicketValidator = kerberosTicketValidator;
         this.threadPool = threadPool;
         this.keytabPath = config.env().configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(config.settings()));
+
+        if (Files.exists(keytabPath) == false) {
+            throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] does not exist");
+        }
+        if (Files.isDirectory(keytabPath)) {
+            throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] is a directory");
+        }
+        if (Files.isReadable(keytabPath) == false) {
+            throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] must have read permission");
+        }
         this.enableKerberosDebug = KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE.get(config.settings());
         this.removeRealmName = KerberosRealmSettings.SETTING_REMOVE_REALM_NAME.get(config.settings());
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java
deleted file mode 100644
index bab899a8664..00000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheck.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.authc.kerberos;
-
-import org.elasticsearch.bootstrap.BootstrapCheck;
-import org.elasticsearch.bootstrap.BootstrapContext;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.xpack.core.security.authc.RealmSettings;
-import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * This class is used to perform bootstrap checks for kerberos realm.
- * <p>
- * We use service keytabs for validating incoming kerberos tickets and is a
- * required configuration. Due to JVM wide system properties for Kerberos we
- * cannot support multiple Kerberos realms. This class adds checks for node to
- * fail if service keytab does not exist or multiple kerberos realms have been
- * configured.
- */
-public class KerberosRealmBootstrapCheck implements BootstrapCheck {
-    private final Environment env;
-
-    public KerberosRealmBootstrapCheck(final Environment env) {
-        this.env = env;
-    }
-
-    @Override
-    public BootstrapCheckResult check(final BootstrapContext context) {
-        final Map<String, Settings> realmsSettings = RealmSettings.getRealmSettings(context.settings);
-        boolean isKerberosRealmConfigured = false;
-        for (final Entry<String, Settings> entry : realmsSettings.entrySet()) {
-            final String name = entry.getKey();
-            final Settings realmSettings = entry.getValue();
-            final String type = realmSettings.get("type");
-            if (Strings.hasText(type) == false) {
-                return BootstrapCheckResult.failure("missing realm type for [" + name + "] realm");
-            }
-            if (KerberosRealmSettings.TYPE.equals(type)) {
-                if (isKerberosRealmConfigured) {
-                    return BootstrapCheckResult.failure(
-                            "multiple [" + type + "] realms are configured. [" + type + "] can only have one such realm configured");
-                }
-                isKerberosRealmConfigured = true;
-
-                final Path keytabPath = env.configFile().resolve(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH.get(realmSettings));
-                if (Files.exists(keytabPath) == false) {
-                    return BootstrapCheckResult.failure("configured service key tab file [" + keytabPath + "] does not exist");
-                }
-            }
-        }
-        return BootstrapCheckResult.success();
-    }
-
-    @Override
-    public boolean alwaysEnforce() {
-        return true;
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
index a71f5cb1cf7..9d795826298 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmsTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -51,13 +52,16 @@ public class RealmsTests extends ESTestCase {
     private XPackLicenseState licenseState;
     private ThreadContext threadContext;
     private ReservedRealm reservedRealm;
+    private int randomRealmTypesCount;
 
     @Before
     public void init() throws Exception {
         factories = new HashMap<>();
         factories.put(FileRealmSettings.TYPE, config -> new DummyRealm(FileRealmSettings.TYPE, config));
         factories.put(NativeRealmSettings.TYPE, config -> new DummyRealm(NativeRealmSettings.TYPE, config));
-        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+        factories.put(KerberosRealmSettings.TYPE, config -> new DummyRealm(KerberosRealmSettings.TYPE, config));
+        randomRealmTypesCount = randomIntBetween(1, 5);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             String name = "type_" + i;
             factories.put(name, config -> new DummyRealm(name, config));
         }
@@ -73,13 +77,13 @@ public class RealmsTests extends ESTestCase {
     public void testWithSettings() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
-        List<Integer> orders = new ArrayList<>(factories.size() - 2);
-        for (int i = 0; i < factories.size() - 2; i++) {
+        List<Integer> orders = new ArrayList<>(randomRealmTypesCount);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             orders.add(i);
         }
         Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
-        for (int i = 0; i < factories.size() - 2; i++) {
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             builder.put("xpack.security.authc.realms.realm_" + i + ".type", "type_" + i);
             builder.put("xpack.security.authc.realms.realm_" + i + ".order", orders.get(i));
             orderToIndex.put(orders.get(i), i);
@@ -107,14 +111,14 @@ public class RealmsTests extends ESTestCase {
     public void testWithSettingsWhereDifferentRealmsHaveSameOrder() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
-        List<Integer> randomSeq = new ArrayList<>(factories.size() - 2);
-        for (int i = 0; i < factories.size() - 2; i++) {
+        List<Integer> randomSeq = new ArrayList<>(randomRealmTypesCount);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             randomSeq.add(i);
         }
         Collections.shuffle(randomSeq, random());
 
         TreeMap<String, Integer> nameToRealmId = new TreeMap<>();
-        for (int i = 0; i < factories.size() - 2; i++) {
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             int randomizedRealmId = randomSeq.get(i);
             String randomizedRealmName = randomAlphaOfLengthBetween(12,32);
             nameToRealmId.put("realm_" + randomizedRealmName, randomizedRealmId);
@@ -181,13 +185,13 @@ public class RealmsTests extends ESTestCase {
     public void testUnlicensedWithOnlyCustomRealms() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
-        List<Integer> orders = new ArrayList<>(factories.size() - 2);
-        for (int i = 0; i < factories.size() - 2; i++) {
+        List<Integer> orders = new ArrayList<>(randomRealmTypesCount);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             orders.add(i);
         }
         Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
-        for (int i = 0; i < factories.size() - 2; i++) {
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             builder.put("xpack.security.authc.realms.realm_" + i + ".type", "type_" + i);
             builder.put("xpack.security.authc.realms.realm_" + i + ".order", orders.get(i));
             orderToIndex.put(orders.get(i), i);
@@ -384,13 +388,13 @@ public class RealmsTests extends ESTestCase {
     public void testDisabledRealmsAreNotAdded() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put("path.home", createTempDir());
-        List<Integer> orders = new ArrayList<>(factories.size() - 2);
-        for (int i = 0; i < factories.size() - 2; i++) {
+        List<Integer> orders = new ArrayList<>(randomRealmTypesCount);
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             orders.add(i);
         }
         Collections.shuffle(orders, random());
         Map<Integer, Integer> orderToIndex = new HashMap<>();
-        for (int i = 0; i < factories.size() - 2; i++) {
+        for (int i = 0; i < randomRealmTypesCount; i++) {
             builder.put("xpack.security.authc.realms.realm_" + i + ".type", "type_" + i);
             builder.put("xpack.security.authc.realms.realm_" + i + ".order", orders.get(i));
             boolean enabled = randomBoolean();
@@ -520,6 +524,20 @@ public class RealmsTests extends ESTestCase {
         }
     }
 
+    public void testInitRealmsFailsForMultipleKerberosRealms() throws IOException {
+        final Settings.Builder builder = Settings.builder().put("path.home", createTempDir());
+        builder.put("xpack.security.authc.realms.realm_1.type", "kerberos");
+        builder.put("xpack.security.authc.realms.realm_1.order", 1);
+        builder.put("xpack.security.authc.realms.realm_2.type", "kerberos");
+        builder.put("xpack.security.authc.realms.realm_2.order", 2);
+        final Settings settings = builder.build();
+        Environment env = TestEnvironment.newEnvironment(settings);
+        final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+                () -> new Realms(settings, env, factories, licenseState, threadContext, reservedRealm));
+        assertThat(iae.getMessage(), is(equalTo(
+                "multiple realms [realm_1, realm_2] configured of type [kerberos], [kerberos] can only have one such realm configured")));
+    }
+
     static class DummyRealm extends Realm {
 
         DummyRealm(String type, RealmConfig config) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java
deleted file mode 100644
index b6e1df9ddbb..00000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmBootstrapCheckTests.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.authc.kerberos;
-
-import org.elasticsearch.bootstrap.BootstrapCheck;
-import org.elasticsearch.bootstrap.BootstrapContext;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.security.authc.RealmSettings;
-import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
-import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
-
-import java.io.IOException;
-import java.nio.file.Path;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class KerberosRealmBootstrapCheckTests extends ESTestCase {
-
-    public void testBootstrapCheckFailsForMultipleKerberosRealms() throws IOException {
-        final Path tempDir = createTempDir();
-        final Settings settings1 = buildKerberosRealmSettings("kerb1", false, tempDir);
-        final Settings settings2 = buildKerberosRealmSettings("kerb2", false, tempDir);
-        final Settings settings3 = realm("pki1", PkiRealmSettings.TYPE, Settings.builder()).build();
-        final Settings settings =
-                Settings.builder().put("path.home", tempDir).put(settings1).put(settings2).put(settings3).build();
-        final BootstrapContext context = new BootstrapContext(settings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isFailure(), is(true));
-        assertThat(result.getMessage(), equalTo("multiple [" + KerberosRealmSettings.TYPE + "] realms are configured. ["
-                + KerberosRealmSettings.TYPE + "] can only have one such realm configured"));
-    }
-
-    public void testBootstrapCheckFailsForMissingKeytabFile() throws IOException {
-        final Path tempDir = createTempDir();
-        final Settings settings =
-                Settings.builder().put("path.home", tempDir).put(buildKerberosRealmSettings("kerb1", true, tempDir)).build();
-        final BootstrapContext context = new BootstrapContext(settings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isFailure(), is(true));
-        assertThat(result.getMessage(),
-                equalTo("configured service key tab file [" + tempDir.resolve("kerb1.keytab").toString() + "] does not exist"));
-    }
-
-    public void testBootstrapCheckFailsForMissingRealmType() throws IOException {
-        final Path tempDir = createTempDir();
-        final String name = "kerb1";
-        final Settings settings1 = buildKerberosRealmSettings("kerb1", false, tempDir);
-        final Settings settings2 = realm(name, randomFrom("", "    "), Settings.builder()).build();
-        final Settings settings =
-                Settings.builder().put("path.home", tempDir).put(settings1).put(settings2).build();
-        final BootstrapContext context = new BootstrapContext(settings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(settings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isFailure(), is(true));
-        assertThat(result.getMessage(), equalTo("missing realm type for [" + name + "] realm"));
-    }
-
-    public void testBootstrapCheckSucceedsForCorrectConfiguration() throws IOException {
-        final Path tempDir = createTempDir();
-        final Settings finalSettings =
-                Settings.builder().put("path.home", tempDir).put(buildKerberosRealmSettings("kerb1", false, tempDir)).build();
-        final BootstrapContext context = new BootstrapContext(finalSettings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(finalSettings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isSuccess(), is(true));
-    }
-
-    public void testBootstrapCheckSucceedsForNoKerberosRealms() throws IOException {
-        final Path tempDir = createTempDir();
-        final Settings finalSettings = Settings.builder().put("path.home", tempDir).build();
-        final BootstrapContext context = new BootstrapContext(finalSettings, null);
-        final KerberosRealmBootstrapCheck kerbRealmBootstrapCheck =
-                new KerberosRealmBootstrapCheck(TestEnvironment.newEnvironment(finalSettings));
-        final BootstrapCheck.BootstrapCheckResult result = kerbRealmBootstrapCheck.check(context);
-        assertThat(result, is(notNullValue()));
-        assertThat(result.isSuccess(), is(true));
-    }
-
-    private Settings buildKerberosRealmSettings(final String name, final boolean missingKeytab, final Path tempDir) throws IOException {
-        final Settings.Builder builder = Settings.builder();
-        if (missingKeytab == false) {
-            KerberosTestCase.writeKeyTab(tempDir.resolve(name + ".keytab"), null);
-        }
-        builder.put(KerberosTestCase.buildKerberosRealmSettings(tempDir.resolve(name + ".keytab").toString()));
-        return realm(name, KerberosRealmSettings.TYPE, builder).build();
-    }
-
-    private Settings.Builder realm(final String name, final String type, final Settings.Builder settings) {
-        final String prefix = RealmSettings.PREFIX + name + ".";
-        if (type != null) {
-            settings.put("type", type);
-        }
-        final Settings.Builder builder = Settings.builder().put(settings.normalizePrefix(prefix).build(), false);
-        return builder;
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
index 43536abaf29..0f972498ab3 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java
@@ -11,17 +11,30 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData;
 import org.ietf.jgss.GSSException;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
 import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.Set;
 
 import javax.security.auth.login.LoginException;
 
@@ -31,6 +44,7 @@ import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.AdditionalMatchers.aryEq;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -94,4 +108,44 @@ public class KerberosRealmTests extends KerberosRealmTestCase {
         assertThat(future.actionGet(), is(nullValue()));
     }
 
+    public void testKerberosRealmWithInvalidKeytabPathConfigurations() throws IOException {
+        final String keytabPathCase = randomFrom("keytabPathAsDirectory", "keytabFileDoesNotExist", "keytabPathWithNoReadPermissions");
+        final String expectedErrorMessage;
+        final String keytabPath;
+        final Set<PosixFilePermission> filePerms;
+        switch (keytabPathCase) {
+        case "keytabPathAsDirectory":
+            final String dirName = randomAlphaOfLength(5);
+            Files.createDirectory(dir.resolve(dirName));
+            keytabPath = dir.resolve(dirName).toString();
+            expectedErrorMessage = "configured service key tab file [" + keytabPath + "] is a directory";
+            break;
+        case "keytabFileDoesNotExist":
+            keytabPath = dir.resolve(randomAlphaOfLength(5) + ".keytab").toString();
+            expectedErrorMessage = "configured service key tab file [" + keytabPath + "] does not exist";
+            break;
+        case "keytabPathWithNoReadPermissions":
+            filePerms = PosixFilePermissions.fromString("---------");
+            final String keytabFileName = randomAlphaOfLength(5) + ".keytab";
+            final FileAttribute<Set<PosixFilePermission>> fileAttributes = PosixFilePermissions.asFileAttribute(filePerms);
+            try (SeekableByteChannel byteChannel = Files.newByteChannel(dir.resolve(keytabFileName),
+                    EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE), fileAttributes)) {
+                byteChannel.write(ByteBuffer.wrap(randomByteArrayOfLength(10)));
+            }
+            keytabPath = dir.resolve(keytabFileName).toString();
+            expectedErrorMessage = "configured service key tab file [" + keytabPath + "] must have read permission";
+            break;
+        default:
+            throw new IllegalArgumentException("Unknown test case :" + keytabPathCase);
+        }
+
+        settings = KerberosTestCase.buildKerberosRealmSettings(keytabPath, 100, "10m", true, randomBoolean());
+        config = new RealmConfig("test-kerb-realm", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings),
+                new ThreadContext(globalSettings));
+        mockNativeRoleMappingStore = roleMappingStore(Arrays.asList("user"));
+        mockKerberosTicketValidator = mock(KerberosTicketValidator.class);
+        final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+                () -> new KerberosRealm(config, mockNativeRoleMappingStore, mockKerberosTicketValidator, threadPool, null));
+        assertThat(iae.getMessage(), is(equalTo(expectedErrorMessage)));
+    }
 }
\ No newline at end of file

From d4ea440e37dcc870c0a9ba63a2c005ae974729b2 Mon Sep 17 00:00:00 2001
From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com>
Date: Tue, 31 Jul 2018 11:18:08 +1000
Subject: [PATCH 233/260] [Kerberos] Add missing javadocs (#32469)

This commit adds missing javadocs and fixes few where
the build failed when using JDK 11 for compilation.

Closes#32461
---
 .../authc/kerberos/KerberosTestCase.java         | 10 +++++-----
 .../authc/kerberos/SimpleKdcLdapServer.java      |  4 ++--
 .../security/authc/kerberos/SpnegoClient.java    | 16 +++++++---------
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
index 891f400c7be..cad9f689d3d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
@@ -133,7 +133,7 @@ public abstract class KerberosTestCase extends ESTestCase {
      * @param dir Directory where the key tab would be created.
      * @param princNames principal names to be created
      * @return {@link Path} to key tab file.
-     * @throws Exception
+     * @throws Exception thrown if principal or keytab could not be created
      */
     protected Path createPrincipalKeyTab(final Path dir, final String... princNames) throws Exception {
         final Path path = dir.resolve(randomAlphaOfLength(10) + ".keytab");
@@ -146,7 +146,7 @@ public abstract class KerberosTestCase extends ESTestCase {
      * 
      * @param principalName Principal name
      * @param password Password
-     * @throws Exception
+     * @throws Exception thrown if principal could not be created
      */
     protected void createPrincipal(final String principalName, final char[] password) throws Exception {
         simpleKdcLdapServer.createPrincipal(principalName, new String(password));
@@ -168,8 +168,8 @@ public abstract class KerberosTestCase extends ESTestCase {
      * @param subject {@link Subject}
      * @param action {@link PrivilegedExceptionAction} action for performing inside
      *            Subject.doAs
-     * @return <T> Type of value as returned by PrivilegedAction
-     * @throws PrivilegedActionException
+     * @return T Type of value as returned by PrivilegedAction
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     static <T> T doAsWrapper(final Subject subject, final PrivilegedExceptionAction<T> action) throws PrivilegedActionException {
         return AccessController.doPrivileged((PrivilegedExceptionAction<T>) () -> Subject.doAs(subject, action));
@@ -181,7 +181,7 @@ public abstract class KerberosTestCase extends ESTestCase {
      * @param keytabPath {@link Path} to keytab file.
      * @param content Content for keytab
      * @return key tab path
-     * @throws IOException
+     * @throws IOException if I/O error occurs while writing keytab file
      */
     public static Path writeKeyTab(final Path keytabPath, final String content) throws IOException {
         try (BufferedWriter bufferedWriter = Files.newBufferedWriter(keytabPath, StandardCharsets.US_ASCII)) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
index 426cacb1a03..53bf9e2b78d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
@@ -66,7 +66,7 @@ public class SimpleKdcLdapServer {
      * @param orgName Org name for base dn
      * @param domainName domain name for base dn
      * @param ldiff for ldap directory.
-     * @throws Exception
+     * @throws Exception when KDC or Ldap server initialization fails
      */
     public SimpleKdcLdapServer(final Path workDir, final String orgName, final String domainName, final Path ldiff) throws Exception {
         this.workDir = workDir;
@@ -194,7 +194,7 @@ public class SimpleKdcLdapServer {
     /**
      * Stop Simple Kdc Server
      * 
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     public synchronized void stop() throws PrivilegedActionException {
         AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
index 1f883b928bd..7fcca3d9f58 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoClient.java
@@ -11,7 +11,6 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidator;
 import org.ietf.jgss.GSSContext;
 import org.ietf.jgss.GSSCredential;
@@ -67,8 +66,8 @@ class SpnegoClient implements AutoCloseable {
      * @param password password for client
      * @param servicePrincipalName Service principal name with whom this client
      *            interacts with.
-     * @throws PrivilegedActionException
-     * @throws GSSException
+     * @throws PrivilegedActionException when privileged action threw exception
+     * @throws GSSException thrown when GSS API error occurs
      */
     SpnegoClient(final String userPrincipalName, final SecureString password, final String servicePrincipalName)
             throws PrivilegedActionException, GSSException {
@@ -99,7 +98,7 @@ class SpnegoClient implements AutoCloseable {
      * base64 encoded token to be sent to server.
      *
      * @return Base64 encoded token
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     String getBase64EncodedTokenForSpnegoHeader() throws PrivilegedActionException {
         final byte[] outToken = KerberosTestCase.doAsWrapper(loginContext.getSubject(),
@@ -114,7 +113,7 @@ class SpnegoClient implements AutoCloseable {
      *            gss negotiation
      * @return Base64 encoded token to be sent to server. May return {@code null} if
      *         nothing to be sent.
-     * @throws PrivilegedActionException
+     * @throws PrivilegedActionException when privileged action threw exception
      */
     String handleResponse(final String base64Token) throws PrivilegedActionException {
         if (gssContext.isEstablished()) {
@@ -160,10 +159,9 @@ class SpnegoClient implements AutoCloseable {
      *
      * @param principal Principal name
      * @param password {@link SecureString}
-     * @param settings {@link Settings}
      * @return authenticated {@link LoginContext} instance. Note: This needs to be
      *         closed {@link LoginContext#logout()} after usage.
-     * @throws LoginException
+     * @throws LoginException thrown if problem with login configuration or when login fails
      */
     private static LoginContext loginUsingPassword(final String principal, final SecureString password) throws LoginException {
         final Set<Principal> principals = Collections.singleton(new KerberosPrincipal(principal));
@@ -182,8 +180,8 @@ class SpnegoClient implements AutoCloseable {
      * Instead of an additional file setting as we do not want the options to be
      * customizable we are constructing it in memory.
      * <p>
-     * As we are uing this instead of jaas.conf, this requires refresh of
-     * {@link Configuration} and reqires appropriate security permissions to do so.
+     * As we are using this instead of jaas.conf, this requires refresh of
+     * {@link Configuration} and requires appropriate security permissions to do so.
      */
     static class PasswordJaasConf extends Configuration {
         private final String principal;

From 8b57e2e5ba7de60f07d411fec0300cb2178790ba Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 31 Jul 2018 08:25:21 +0100
Subject: [PATCH 234/260] Fix calculation of orientation of polygons (#27967)

The method for working out whether a polygon is clockwise or anticlockwise is
mostly correct but doesn't work in some rare cases such as the included test
case. This commit fixes that.
---
 .../common/geo/builders/PolygonBuilder.java   | 41 +++++++++++++++----
 .../geo/builders/PolygonBuilderTests.java     | 18 ++++++++
 2 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
index dbcb46d9936..e4d48b8df45 100644
--- a/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
+++ b/server/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
@@ -46,6 +46,8 @@ import java.util.Locale;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.apache.lucene.geo.GeoUtils.orient;
+
 /**
  * The {@link PolygonBuilder} implements the groundwork to create polygons. This contains
  * Methods to wrap polygons at the dateline and building shapes from the data held by the
@@ -642,14 +644,8 @@ public class PolygonBuilder extends ShapeBuilder<JtsGeometry, PolygonBuilder> {
      */
     private static Edge[] ring(int component, boolean direction, boolean handedness,
                                  Coordinate[] points, int offset, Edge[] edges, int toffset, int length, final AtomicBoolean translated) {
-        // calculate the direction of the points:
-        // find the point a the top of the set and check its
-        // neighbors orientation. So direction is equivalent
-        // to clockwise/counterclockwise
-        final int top = top(points, offset, length);
-        final int prev = (offset + ((top + length - 1) % length));
-        final int next = (offset + ((top + 1) % length));
-        boolean orientation = points[offset + prev].x > points[offset + next].x;
+
+        boolean orientation = getOrientation(points, offset, length);
 
         // OGC requires shell as ccw (Right-Handedness) and holes as cw (Left-Handedness)
         // since GeoJSON doesn't specify (and doesn't need to) GEO core will assume OGC standards
@@ -678,6 +674,35 @@ public class PolygonBuilder extends ShapeBuilder<JtsGeometry, PolygonBuilder> {
         return concat(component, direction ^ orientation, points, offset, edges, toffset, length);
     }
 
+    /**
+     * @return whether the points are clockwise (true) or anticlockwise (false)
+     */
+    private static boolean getOrientation(Coordinate[] points, int offset, int length) {
+        // calculate the direction of the points: find the southernmost point
+        // and check its neighbors orientation.
+
+        final int top = top(points, offset, length);
+        final int prev = (top + length - 1) % length;
+        final int next = (top + 1) % length;
+
+        final int determinantSign = orient(
+            points[offset + prev].x, points[offset + prev].y,
+            points[offset + top].x, points[offset + top].y,
+            points[offset + next].x, points[offset + next].y);
+
+        if (determinantSign == 0) {
+            // Points are collinear, but `top` is not in the middle if so, so the edges either side of `top` are intersecting.
+            throw new InvalidShapeException("Cannot determine orientation: edges adjacent to ("
+                + points[offset + top].x + "," + points[offset +top].y + ") coincide");
+        }
+
+        return determinantSign < 0;
+    }
+
+    /**
+     * @return the (offset) index of the point that is furthest west amongst
+     * those points that are the furthest south in the set.
+     */
     private static int top(Coordinate[] points, int offset, int length) {
         int top = 0; // we start at 1 here since top points to 0
         for (int i = 1; i < length; i++) {
diff --git a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
index 7f8b893caf0..83bf2bae6a6 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
@@ -143,4 +143,22 @@ public class PolygonBuilderTests extends AbstractShapeBuilderTestCase<PolygonBui
 
         assertEquals("Hole lies outside shell at or near point (4.0, 3.0, NaN)", e.getMessage());
     }
+
+    public void testWidePolygonWithConfusingOrientation() {
+        // A valid polygon that is oriented correctly (anticlockwise) but which
+        // confounds a naive algorithm for determining its orientation leading
+        // ES to believe that it crosses the dateline and "fixing" it in a way
+        // that self-intersects.
+
+        PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder()
+            .coordinate(10, -20).coordinate(100, 0).coordinate(-100, 0).coordinate(20, -45).coordinate(40, -60).close());
+        pb.build(); // Should not throw an exception
+    }
+
+    public void testPolygonWithUndefinedOrientationDueToCollinearPoints() {
+        PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder()
+            .coordinate(0.0, 0.0).coordinate(1.0, 1.0).coordinate(-1.0, -1.0).close());
+        InvalidShapeException e = expectThrows(InvalidShapeException.class, pb::build);
+        assertEquals("Cannot determine orientation: edges adjacent to (-1.0,-1.0) coincide", e.getMessage());
+    }
 }

From a3b272966d8de964251186cb06967e5ec7e66f89 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Tue, 31 Jul 2018 09:41:51 +0200
Subject: [PATCH 235/260] High-level client: fix clusterAlias parsing in
 SearchHit (#32465)

When using cross-cluster search through the high-level REST client, the cluster alias from each search hit was not parsed correctly. It would be part of the index field initially, but overridden just a few lines later once setting the shard target (in case we have enough info to build it from the response). In any case, getClusterAlias returns `null` which is a bug.

With this change we rather parse back clusterAliases from the index name, set its corresponding field and properly handle the two possible cases depending on whether we can or cannot build the shard target object.
---
 .../org/elasticsearch/search/SearchHit.java   | 57 ++++++++++++-------
 .../search/SearchShardTarget.java             |  6 +-
 .../elasticsearch/search/SearchHitTests.java  |  5 +-
 3 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java
index 66999c7e389..28a600c0d21 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -19,6 +19,16 @@
 
 package org.elasticsearch.search;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 import org.apache.lucene.search.Explanation;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.OriginalIndices;
@@ -51,16 +61,6 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.transport.RemoteClusterAware;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static java.util.Collections.unmodifiableMap;
@@ -107,6 +107,9 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable<D
     @Nullable
     private SearchShardTarget shard;
 
+    //These two fields normally get set when setting the shard target, so they hold the same values as the target thus don't get
+    //serialized over the wire. When parsing hits back from xcontent though, in most of the cases (whenever explanation is disabled)
+    //we can't rebuild the shard target object so we need to set these manually for users retrieval.
     private transient String index;
     private transient String clusterAlias;
 
@@ -551,7 +554,26 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable<D
         Map<String, DocumentField> fields = get(Fields.FIELDS, values, Collections.emptyMap());
 
         SearchHit searchHit = new SearchHit(-1, id, type, nestedIdentity, fields);
-        searchHit.index = get(Fields._INDEX, values, null);
+        String index = get(Fields._INDEX, values, null);
+        String clusterAlias = null;
+        if (index != null) {
+            int indexOf = index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR);
+            if (indexOf > 0) {
+                clusterAlias = index.substring(0, indexOf);
+                index = index.substring(indexOf + 1);
+            }
+        }
+        ShardId shardId = get(Fields._SHARD, values, null);
+        String nodeId = get(Fields._NODE, values, null);
+        if (shardId != null && nodeId != null) {
+            assert shardId.getIndexName().equals(index);
+            searchHit.shard(new SearchShardTarget(nodeId, shardId, clusterAlias, OriginalIndices.NONE));
+        } else {
+            //these fields get set anyways when setting the shard target,
+            //but we set them explicitly when we don't have enough info to rebuild the shard target
+            searchHit.index = index;
+            searchHit.clusterAlias = clusterAlias;
+        }
         searchHit.score(get(Fields._SCORE, values, DEFAULT_SCORE));
         searchHit.version(get(Fields._VERSION, values, -1L));
         searchHit.sortValues(get(Fields.SORT, values, SearchSortValues.EMPTY));
@@ -561,12 +583,7 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable<D
         searchHit.setInnerHits(get(Fields.INNER_HITS, values, null));
         List<String> matchedQueries = get(Fields.MATCHED_QUERIES, values, null);
         if (matchedQueries != null) {
-            searchHit.matchedQueries(matchedQueries.toArray(new String[matchedQueries.size()]));
-        }
-        ShardId shardId = get(Fields._SHARD, values, null);
-        String nodeId = get(Fields._NODE, values, null);
-        if (shardId != null && nodeId != null) {
-            searchHit.shard(new SearchShardTarget(nodeId, shardId, null, OriginalIndices.NONE));
+            searchHit.matchedQueries(matchedQueries.toArray(new String[0]));
         }
         return searchHit;
     }
@@ -842,13 +859,15 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable<D
                 && Arrays.equals(matchedQueries, other.matchedQueries)
                 && Objects.equals(explanation, other.explanation)
                 && Objects.equals(shard, other.shard)
-                && Objects.equals(innerHits, other.innerHits);
+                && Objects.equals(innerHits, other.innerHits)
+                && Objects.equals(index, other.index)
+                && Objects.equals(clusterAlias, other.clusterAlias);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(id, type, nestedIdentity, version, source, fields, getHighlightFields(), Arrays.hashCode(matchedQueries),
-                explanation, shard, innerHits);
+                explanation, shard, innerHits, index, clusterAlias);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java
index 4ac742ff5d9..faf415b54ae 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchShardTarget.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.search;
 
+import java.io.IOException;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Nullable;
@@ -30,8 +32,6 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.transport.RemoteClusterAware;
 
-import java.io.IOException;
-
 /**
  * The target that the search request was executed on.
  */
@@ -39,7 +39,7 @@ public final class SearchShardTarget implements Writeable, Comparable<SearchShar
 
     private final Text nodeId;
     private final ShardId shardId;
-    //original indices and cluster alias are only needed in the coordinating node throughout the search request execution.
+    //original indices are only needed in the coordinating node throughout the search request execution.
     //no need to serialize them as part of SearchShardTarget.
     private final transient OriginalIndices originalIndices;
     private final String clusterAlias;
diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
index 87b8ba2dc59..996faf03ffb 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
@@ -119,9 +119,10 @@ public class SearchHitTests extends ESTestCase {
             hit.setInnerHits(innerHits);
         }
         if (randomBoolean()) {
+            String index = randomAlphaOfLengthBetween(5, 10);
+            String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 10);
             hit.shard(new SearchShardTarget(randomAlphaOfLengthBetween(5, 10),
-                    new ShardId(new Index(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)), randomInt()), null,
-                    OriginalIndices.NONE));
+                new ShardId(new Index(index, randomAlphaOfLengthBetween(5, 10)), randomInt()), clusterAlias, OriginalIndices.NONE));
         }
         return hit;
     }

From 11f6983d6919a694e0c8c623b3f711690a373006 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 31 Jul 2018 10:29:22 +0200
Subject: [PATCH 236/260] HLRC: Add delete watch action (#32337)

Adds the "delete watch" API to the High-Level Rest Client.

Relates #29827
---
 .../client/RequestConverters.java             |  13 ++
 .../elasticsearch/client/WatcherClient.java   |  30 +++++
 .../client/RequestConvertersTests.java        |  12 ++
 .../org/elasticsearch/client/WatcherIT.java   |  37 +++++-
 .../documentation/WatcherDocumentationIT.java |  45 ++++++-
 .../high-level/supported-apis.asciidoc        |   3 +
 .../x-pack/watcher/delete-watch.asciidoc      |  53 ++++++++
 .../x-pack/watcher/put-watch.asciidoc         |   8 +-
 .../core/watcher/client/WatcherClient.java    |   4 +-
 .../actions/delete/DeleteWatchAction.java     |   1 +
 .../delete/DeleteWatchRequestBuilder.java     |   2 +
 .../actions/delete/DeleteWatchResponse.java   |  56 --------
 .../exporter/local/LocalExporter.java         |   2 +-
 .../rest/action/RestDeleteWatchAction.java    |   4 +-
 .../delete/TransportDeleteWatchAction.java    |   4 +-
 .../test/integration/BasicWatcherTests.java   |   2 +-
 .../action/WatchRequestValidationTests.java   |   2 +-
 .../action/delete/DeleteWatchTests.java       |   2 +-
 .../xpack/watcher}/DeleteWatchRequest.java    |  24 +++-
 .../xpack/watcher/DeleteWatchResponse.java    | 123 ++++++++++++++++++
 .../watcher/DeleteWatchResponseTests.java     |  45 +++++++
 21 files changed, 391 insertions(+), 81 deletions(-)
 create mode 100644 docs/java-rest/high-level/x-pack/watcher/delete-watch.asciidoc
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchResponse.java
 rename x-pack/{plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete => protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher}/DeleteWatchRequest.java (67%)
 create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponse.java
 create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index b9360877dfc..ce6fd1c8c94 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -107,6 +107,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
@@ -1133,6 +1134,18 @@ final class RequestConverters {
         return request;
     }
 
+    static Request xPackWatcherDeleteWatch(DeleteWatchRequest deleteWatchRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("watcher")
+            .addPathPartAsIs("watch")
+            .addPathPart(deleteWatchRequest.getId())
+            .build();
+
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+        return request;
+    }
+
     static Request xpackUsage(XPackUsageRequest usageRequest) {
         Request request = new Request(HttpGet.METHOD_NAME, "/_xpack/usage");
         Params parameters = new Params(request);
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
index 73c92ba5c45..48487926f02 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherClient.java
@@ -19,12 +19,15 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 
 import java.io.IOException;
 
 import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
 
 public final class WatcherClient {
 
@@ -61,4 +64,31 @@ public final class WatcherClient {
         restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xPackWatcherPutWatch, options,
             PutWatchResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Deletes a watch from the cluster
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-delete-watch.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public DeleteWatchResponse deleteWatch(DeleteWatchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::xPackWatcherDeleteWatch, options,
+            DeleteWatchResponse::fromXContent, singleton(404));
+    }
+
+    /**
+     * Asynchronously deletes a watch from the cluster
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-delete-watch.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void deleteWatchAsync(DeleteWatchRequest request, RequestOptions options, ActionListener<DeleteWatchResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xPackWatcherDeleteWatch, options,
+            DeleteWatchResponse::fromXContent, listener, singleton(404));
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index 0415d363c54..e4aa690acb6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -126,6 +126,7 @@ import org.elasticsearch.index.rankeval.RankEvalSpec;
 import org.elasticsearch.index.rankeval.RatedRequest;
 import org.elasticsearch.index.rankeval.RestRankEvalAction;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.action.search.RestSearchAction;
@@ -2580,6 +2581,17 @@ public class RequestConvertersTests extends ESTestCase {
         assertThat(bos.toString("UTF-8"), is(body));
     }
 
+    public void testXPackDeleteWatch() {
+        DeleteWatchRequest deleteWatchRequest = new DeleteWatchRequest();
+        String watchId = randomAlphaOfLength(10);
+        deleteWatchRequest.setId(watchId);
+
+        Request request = RequestConverters.xPackWatcherDeleteWatch(deleteWatchRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_xpack/watcher/watch/" + watchId, request.getEndpoint());
+        assertThat(request.getEntity(), nullValue());
+    }
+
     /**
      * Randomize the {@link FetchSourceContext} request parameters.
      */
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
index dec438a47ab..67d1def323a 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/WatcherIT.java
@@ -21,6 +21,8 @@ package org.elasticsearch.client;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 
@@ -30,6 +32,13 @@ public class WatcherIT extends ESRestHighLevelClientTestCase {
 
     public void testPutWatch() throws Exception {
         String watchId = randomAlphaOfLength(10);
+        PutWatchResponse putWatchResponse = createWatch(watchId);
+        assertThat(putWatchResponse.isCreated(), is(true));
+        assertThat(putWatchResponse.getId(), is(watchId));
+        assertThat(putWatchResponse.getVersion(), is(1L));
+    }
+
+    private PutWatchResponse createWatch(String watchId) throws Exception {
         String json = "{ \n" +
             "  \"trigger\": { \"schedule\": { \"interval\": \"10h\" } },\n" +
             "  \"input\": { \"none\": {} },\n" +
@@ -37,10 +46,30 @@ public class WatcherIT extends ESRestHighLevelClientTestCase {
             "}";
         BytesReference bytesReference = new BytesArray(json);
         PutWatchRequest putWatchRequest = new PutWatchRequest(watchId, bytesReference, XContentType.JSON);
-        PutWatchResponse putWatchResponse = highLevelClient().xpack().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT);
-        assertThat(putWatchResponse.isCreated(), is(true));
-        assertThat(putWatchResponse.getId(), is(watchId));
-        assertThat(putWatchResponse.getVersion(), is(1L));
+        return highLevelClient().xpack().watcher().putWatch(putWatchRequest, RequestOptions.DEFAULT);
+    }
+
+    public void testDeleteWatch() throws Exception {
+        // delete watch that exists
+        {
+            String watchId = randomAlphaOfLength(10);
+            createWatch(watchId);
+            DeleteWatchResponse deleteWatchResponse = highLevelClient().xpack().watcher().deleteWatch(new DeleteWatchRequest(watchId),
+                RequestOptions.DEFAULT);
+            assertThat(deleteWatchResponse.getId(), is(watchId));
+            assertThat(deleteWatchResponse.getVersion(), is(2L));
+            assertThat(deleteWatchResponse.isFound(), is(true));
+        }
+
+        // delete watch that does not exist
+        {
+            String watchId = randomAlphaOfLength(10);
+            DeleteWatchResponse deleteWatchResponse = highLevelClient().xpack().watcher().deleteWatch(new DeleteWatchRequest(watchId),
+                RequestOptions.DEFAULT);
+            assertThat(deleteWatchResponse.getId(), is(watchId));
+            assertThat(deleteWatchResponse.getVersion(), is(1L));
+            assertThat(deleteWatchResponse.isFound(), is(false));
+        }
     }
 
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
index df51d896cda..47f8510b746 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java
@@ -26,6 +26,8 @@ import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 
@@ -34,7 +36,7 @@ import java.util.concurrent.TimeUnit;
 
 public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase {
 
-    public void testPutWatch() throws Exception {
+    public void testWatcher() throws Exception {
         RestHighLevelClient client = highLevelClient();
 
         {
@@ -88,5 +90,46 @@ public class WatcherDocumentationIT extends ESRestHighLevelClientTestCase {
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
         }
+
+        {
+            //tag::x-pack-delete-watch-execute
+            DeleteWatchRequest request = new DeleteWatchRequest("my_watch_id");
+            DeleteWatchResponse response = client.xpack().watcher().deleteWatch(request, RequestOptions.DEFAULT);
+            //end::x-pack-delete-watch-execute
+
+            //tag::x-pack-delete-watch-response
+            String watchId = response.getId(); // <1>
+            boolean found = response.isFound(); // <2>
+            long version = response.getVersion(); // <3>
+            //end::x-pack-delete-watch-response
+        }
+
+        {
+            DeleteWatchRequest request = new DeleteWatchRequest("my_other_watch_id");
+            // tag::x-pack-delete-watch-execute-listener
+            ActionListener<DeleteWatchResponse> listener = new ActionListener<DeleteWatchResponse>() {
+                @Override
+                public void onResponse(DeleteWatchResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::x-pack-delete-watch-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::x-pack-delete-watch-execute-async
+            client.xpack().watcher().deleteWatchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::x-pack-delete-watch-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
     }
+
 }
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index 25fbcaaaeaa..63aef865955 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -54,11 +54,14 @@ The Java High Level REST Client supports the following Miscellaneous APIs:
 * <<java-rest-high-main>>
 * <<java-rest-high-ping>>
 * <<java-rest-high-x-pack-info>>
+* <<java-rest-high-x-pack-watcher-put-watch>>
+* <<java-rest-high-x-pack-watcher-delete-watch>>
 
 include::miscellaneous/main.asciidoc[]
 include::miscellaneous/ping.asciidoc[]
 include::x-pack/x-pack-info.asciidoc[]
 include::x-pack/watcher/put-watch.asciidoc[]
+include::x-pack/watcher/delete-watch.asciidoc[]
 
 == Indices APIs
 
diff --git a/docs/java-rest/high-level/x-pack/watcher/delete-watch.asciidoc b/docs/java-rest/high-level/x-pack/watcher/delete-watch.asciidoc
new file mode 100644
index 00000000000..d5f35817558
--- /dev/null
+++ b/docs/java-rest/high-level/x-pack/watcher/delete-watch.asciidoc
@@ -0,0 +1,53 @@
+[[java-rest-high-x-pack-watcher-delete-watch]]
+=== X-Pack Delete Watch API
+
+[[java-rest-high-x-pack-watcher-delete-watch-execution]]
+==== Execution
+
+A watch can be deleted as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-delete-watch-execute]
+--------------------------------------------------
+
+[[java-rest-high-x-pack-watcher-delete-watch-response]]
+==== Response
+
+The returned `DeleteWatchResponse` contains `found`, `id`,
+and `version` information.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-response]
+--------------------------------------------------
+<1> `_id` contains id of the watch
+<2> `found` is a boolean indicating whether the watch was found
+<3> `_version` returns the version of the deleted watch
+
+[[java-rest-high-x-pack-watcher-delete-watch-async]]
+==== Asynchronous Execution
+
+This request can be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-delete-watch-execute-async]
+--------------------------------------------------
+<1> The `DeleteWatchRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The asynchronous method does not block and returns immediately. Once it is
+completed the `ActionListener` is called back using the `onResponse` method
+if the execution successfully completed or using the `onFailure` method if
+it failed.
+
+A typical listener for `DeleteWatchResponse` looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-delete-watch-execute-listener]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
diff --git a/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc b/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc
index c803c54eb5e..a76ba407a1a 100644
--- a/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc
+++ b/docs/java-rest/high-level/x-pack/watcher/put-watch.asciidoc
@@ -1,5 +1,5 @@
 [[java-rest-high-x-pack-watcher-put-watch]]
-=== X-Pack Info API
+=== X-Pack Put Watch API
 
 [[java-rest-high-x-pack-watcher-put-watch-execution]]
 ==== Execution
@@ -16,7 +16,7 @@ include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-execute
 [[java-rest-high-x-pack-watcher-put-watch-response]]
 ==== Response
 
-The returned `XPackPutWatchResponse` contain `created`, `id`,
+The returned `PutWatchResponse` contains `created`, `id`,
 and `version` information.
 
 ["source","java",subs="attributes,callouts,macros"]
@@ -36,7 +36,7 @@ This request can be executed asynchronously:
 --------------------------------------------------
 include-tagged::{doc-tests}/WatcherDocumentationIT.java[x-pack-put-watch-execute-async]
 --------------------------------------------------
-<1> The `XPackPutWatchRequest` to execute and the `ActionListener` to use when
+<1> The `PutWatchRequest` to execute and the `ActionListener` to use when
 the execution completes
 
 The asynchronous method does not block and returns immediately. Once it is
@@ -44,7 +44,7 @@ completed the `ActionListener` is called back using the `onResponse` method
 if the execution successfully completed or using the `onFailure` method if
 it failed.
 
-A typical listener for `XPackPutWatchResponse` looks like:
+A typical listener for `PutWatchResponse` looks like:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
index 063f1f655a4..b9458c23ddc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
@@ -19,9 +19,9 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateW
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
index 8a16755a6db..eb440ddc251 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.watcher.transport.actions.delete;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 
 /**
  * This action deletes an watch from in memory, the scheduler and the index
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java
index afea505aa4a..6779eec1ddf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequestBuilder.java
@@ -7,6 +7,8 @@ package org.elasticsearch.xpack.core.watcher.transport.actions.delete;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 
 /**
  * A delete document action request builder.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchResponse.java
deleted file mode 100644
index bda5ecabaa0..00000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchResponse.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.watcher.transport.actions.delete;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-
-public class DeleteWatchResponse extends ActionResponse {
-
-    private String id;
-    private long version;
-    private boolean found;
-
-    public DeleteWatchResponse() {
-    }
-
-    public DeleteWatchResponse(String id, long version, boolean found) {
-        this.id = id;
-        this.version = version;
-        this.found = found;
-    }
-
-    public String getId() {
-        return id;
-    }
-
-    public long getVersion() {
-        return version;
-    }
-
-    public boolean isFound() {
-        return found;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        id = in.readString();
-        version = in.readVLong();
-        found = in.readBoolean();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeString(id);
-        out.writeVLong(version);
-        out.writeBoolean(found);
-    }
-}
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
index 08618b45cf5..6183dd8665f 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
@@ -46,7 +46,7 @@ import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
index e41d52c1954..28eb884bbe8 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
@@ -14,8 +14,8 @@ import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/delete/TransportDeleteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/delete/TransportDeleteWatchAction.java
index d7ff25b623f..9cc94820554 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/delete/TransportDeleteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/delete/TransportDeleteWatchAction.java
@@ -18,8 +18,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 
 import java.util.function.Supplier;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
index 18f151b4f6a..c07e6ba3db1 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
@@ -19,7 +19,7 @@ import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
index 64b913f96a0..62985cad1f1 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/WatchRequestValidationTests.java
@@ -14,7 +14,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
index 76f71b9c95e..2f502cb95aa 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
@@ -13,7 +13,7 @@ import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchResponse;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchRequest.java
similarity index 67%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequest.java
rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchRequest.java
index f8c1a71f1eb..1ec83a8c05a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchRequest.java
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchRequest.java
@@ -1,9 +1,22 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
-package org.elasticsearch.xpack.core.watcher.transport.actions.delete;
+package org.elasticsearch.protocol.xpack.watcher;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -11,7 +24,6 @@ import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.xpack.core.watcher.support.WatcherUtils;
 
 import java.io.IOException;
 
@@ -50,7 +62,7 @@ public class DeleteWatchRequest extends ActionRequest {
         ActionRequestValidationException validationException = null;
         if (id == null){
             validationException = ValidateActions.addValidationError("watch id is missing", validationException);
-        } else if (WatcherUtils.isValidId(id) == false) {
+        } else if (PutWatchRequest.isValidId(id) == false) {
             validationException = ValidateActions.addValidationError("watch id contains whitespace", validationException);
         }
         return validationException;
diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponse.java
new file mode 100644
index 00000000000..b644a6a854c
--- /dev/null
+++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponse.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.watcher;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DeleteWatchResponse extends ActionResponse implements ToXContentObject {
+
+    private static final ObjectParser<DeleteWatchResponse, Void> PARSER
+        = new ObjectParser<>("x_pack_delete_watch_response", DeleteWatchResponse::new);
+    static {
+        PARSER.declareString(DeleteWatchResponse::setId, new ParseField("_id"));
+        PARSER.declareLong(DeleteWatchResponse::setVersion, new ParseField("_version"));
+        PARSER.declareBoolean(DeleteWatchResponse::setFound, new ParseField("found"));
+    }
+
+    private String id;
+    private long version;
+    private boolean found;
+
+    public DeleteWatchResponse() {
+    }
+
+    public DeleteWatchResponse(String id, long version, boolean found) {
+        this.id = id;
+        this.version = version;
+        this.found = found;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    public boolean isFound() {
+        return found;
+    }
+
+    private void setId(String id) {
+        this.id = id;
+    }
+
+    private void setVersion(long version) {
+        this.version = version;
+    }
+
+    private void setFound(boolean found) {
+        this.found = found;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DeleteWatchResponse that = (DeleteWatchResponse) o;
+
+        return Objects.equals(id, that.id) && Objects.equals(version, that.version) && Objects.equals(found, that.found);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, version, found);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        id = in.readString();
+        version = in.readVLong();
+        found = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(id);
+        out.writeVLong(version);
+        out.writeBoolean(found);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject()
+            .field("_id", id)
+            .field("_version", version)
+            .field("found", found)
+            .endObject();
+    }
+
+    public static DeleteWatchResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+}
diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java
new file mode 100644
index 00000000000..1dbc4cec321
--- /dev/null
+++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.watcher;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class DeleteWatchResponseTests extends AbstractXContentTestCase<DeleteWatchResponse> {
+
+    @Override
+    protected DeleteWatchResponse createTestInstance() {
+        String id = randomAlphaOfLength(10);
+        long version = randomLongBetween(1, 10);
+        boolean found = randomBoolean();
+        return new DeleteWatchResponse(id, version, found);
+    }
+
+    @Override
+    protected DeleteWatchResponse doParseInstance(XContentParser parser) throws IOException {
+        return DeleteWatchResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}

From 9703d06321913dab9e59a91b5e4aef41fede6b29 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel@mitterdorfer.name>
Date: Tue, 31 Jul 2018 13:28:49 +0200
Subject: [PATCH 237/260] Mute QueryProfilerIT#testProfileMatchesRegular()

Relates #32492
---
 .../org/elasticsearch/search/profile/query/QueryProfilerIT.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
index 6de15146fae..4f725ed2f11 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
@@ -105,6 +105,7 @@ public class QueryProfilerIT extends ESIntegTestCase {
      * search for each query.  It then does some basic sanity checking of score and hits
      * to make sure the profiling doesn't interfere with the hits being returned
      */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32492")
     public void testProfileMatchesRegular() throws Exception {
         createIndex("test");
         ensureGreen();

From 97b379e0d4fdcf04c769788644676f9555e1f672 Mon Sep 17 00:00:00 2001
From: Colm O'Shea <colmose@gmail.com>
Date: Tue, 31 Jul 2018 13:33:23 +0100
Subject: [PATCH 238/260] fix no=>not typo (#32463)

Found a tiny typo while reading the docs
---
 docs/reference/aggregations/bucket/terms-aggregation.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 551c082a662..f42d176aea0 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -653,7 +653,7 @@ GET /_search
 // CONSOLE
 
 In the above example, buckets will be created for all the tags that has the word `sport` in them, except those starting
-with `water_` (so the tag `water_sports` will no be aggregated). The `include` regular expression will determine what
+with `water_` (so the tag `water_sports` will not be aggregated). The `include` regular expression will determine what
 values are "allowed" to be aggregated, while the `exclude` determines the values that should not be aggregated. When
 both are defined, the `exclude` has precedence, meaning, the `include` is evaluated first and only then the `exclude`.
 

From 6f93911955dfe0e694110271cb9dd0ab2991240d Mon Sep 17 00:00:00 2001
From: Paul Sanwald <paul.sanwald@elastic.co>
Date: Tue, 31 Jul 2018 08:52:16 -0400
Subject: [PATCH 239/260] Fix AutoIntervalDateHistogram.testReduce random
 failures (#32301)

1. Refactor the test to use the same roundings as the implementation.
2. Refactor the test verification logic to use `innerIntervals` when rounding.
---
 .../AutoDateHistogramAggregationBuilder.java  | 41 ++++++++----
 .../histogram/InternalAutoDateHistogram.java  |  8 ++-
 .../InternalAutoDateHistogramTests.java       | 64 +++++++++++++------
 3 files changed, 78 insertions(+), 35 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
index 366060835d8..50a0c85c041 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
@@ -42,6 +42,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.internal.SearchContext;
+import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -53,7 +54,7 @@ public class AutoDateHistogramAggregationBuilder
 
     public static final String NAME = "auto_date_histogram";
 
-    public static final ParseField NUM_BUCKETS_FIELD = new ParseField("buckets");
+    private static final ParseField NUM_BUCKETS_FIELD = new ParseField("buckets");
 
     private static final ObjectParser<AutoDateHistogramAggregationBuilder, Void> PARSER;
     static {
@@ -63,6 +64,29 @@ public class AutoDateHistogramAggregationBuilder
         PARSER.declareInt(AutoDateHistogramAggregationBuilder::setNumBuckets, NUM_BUCKETS_FIELD);
     }
 
+    /**
+     *
+     * Build roundings, computed dynamically as roundings are time zone dependent.
+     * The current implementation probably should not be invoked in a tight loop.
+     * @return Array of RoundingInfo
+     */
+    static RoundingInfo[] buildRoundings(DateTimeZone timeZone) {
+        RoundingInfo[] roundings = new RoundingInfo[6];
+        roundings[0] = new RoundingInfo(createRounding(DateTimeUnit.SECOND_OF_MINUTE, timeZone),
+            1000L, 1, 5, 10, 30);
+        roundings[1] = new RoundingInfo(createRounding(DateTimeUnit.MINUTES_OF_HOUR, timeZone),
+            60 * 1000L, 1, 5, 10, 30);
+        roundings[2] = new RoundingInfo(createRounding(DateTimeUnit.HOUR_OF_DAY, timeZone),
+            60 * 60 * 1000L, 1, 3, 12);
+        roundings[3] = new RoundingInfo(createRounding(DateTimeUnit.DAY_OF_MONTH, timeZone),
+            24 * 60 * 60 * 1000L, 1, 7);
+        roundings[4] = new RoundingInfo(createRounding(DateTimeUnit.MONTH_OF_YEAR, timeZone),
+            30 * 24 * 60 * 60 * 1000L, 1, 3);
+        roundings[5] = new RoundingInfo(createRounding(DateTimeUnit.YEAR_OF_CENTURY, timeZone),
+            365 * 24 * 60 * 60 * 1000L, 1, 5, 10, 20, 50, 100);
+        return roundings;
+    }
+
     public static AutoDateHistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
         return PARSER.parse(parser, new AutoDateHistogramAggregationBuilder(aggregationName), null);
     }
@@ -116,14 +140,7 @@ public class AutoDateHistogramAggregationBuilder
     @Override
     protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        RoundingInfo[] roundings = new RoundingInfo[6];
-        roundings[0] = new RoundingInfo(createRounding(DateTimeUnit.SECOND_OF_MINUTE), 1000L, 1, 5, 10, 30);
-        roundings[1] = new RoundingInfo(createRounding(DateTimeUnit.MINUTES_OF_HOUR), 60 * 1000L, 1, 5, 10, 30);
-        roundings[2] = new RoundingInfo(createRounding(DateTimeUnit.HOUR_OF_DAY), 60 * 60 * 1000L, 1, 3, 12);
-        roundings[3] = new RoundingInfo(createRounding(DateTimeUnit.DAY_OF_MONTH), 24 * 60 * 60 * 1000L, 1, 7);
-        roundings[4] = new RoundingInfo(createRounding(DateTimeUnit.MONTH_OF_YEAR), 30 * 24 * 60 * 60 * 1000L, 1, 3);
-        roundings[5] = new RoundingInfo(createRounding(DateTimeUnit.YEAR_OF_CENTURY), 365 * 24 * 60 * 60 * 1000L, 1, 5, 10, 20, 50, 100);
-
+        RoundingInfo[] roundings = buildRoundings(timeZone());
         int maxRoundingInterval = Arrays.stream(roundings,0, roundings.length-1)
             .map(rounding -> rounding.innerIntervals)
             .flatMapToInt(Arrays::stream)
@@ -139,10 +156,10 @@ public class AutoDateHistogramAggregationBuilder
         return new AutoDateHistogramAggregatorFactory(name, config, numBuckets, roundings, context, parent, subFactoriesBuilder, metaData);
     }
 
-    private Rounding createRounding(DateTimeUnit interval) {
+    private static Rounding createRounding(DateTimeUnit interval, DateTimeZone timeZone) {
         Rounding.Builder tzRoundingBuilder = Rounding.builder(interval);
-        if (timeZone() != null) {
-            tzRoundingBuilder.timeZone(timeZone());
+        if (timeZone != null) {
+            tzRoundingBuilder.timeZone(timeZone);
         }
         Rounding rounding = tzRoundingBuilder.build();
         return rounding;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
index 27c195cbdae..6a78ca67249 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
@@ -447,7 +447,8 @@ public final class InternalAutoDateHistogram extends
         return new BucketReduceResult(list, roundingInfo, roundingIdx);
     }
 
-    private int getAppropriateRounding(long minKey, long maxKey, int roundingIdx, RoundingInfo[] roundings) {
+    private int getAppropriateRounding(long minKey, long maxKey, int roundingIdx,
+                                              RoundingInfo[] roundings) {
         if (roundingIdx == roundings.length - 1) {
             return roundingIdx;
         }
@@ -509,7 +510,8 @@ public final class InternalAutoDateHistogram extends
                 pipelineAggregators(), getMetaData());
     }
 
-    private BucketReduceResult maybeMergeConsecutiveBuckets(BucketReduceResult reducedBucketsResult, ReduceContext reduceContext) {
+    private BucketReduceResult maybeMergeConsecutiveBuckets(BucketReduceResult reducedBucketsResult,
+                                                                   ReduceContext reduceContext) {
         List<Bucket> buckets = reducedBucketsResult.buckets;
         RoundingInfo roundingInfo = reducedBucketsResult.roundingInfo;
         int roundingIdx = reducedBucketsResult.roundingIdx;
@@ -539,7 +541,7 @@ public final class InternalAutoDateHistogram extends
                 key = roundingInfo.rounding.round(bucket.key);
             }
             reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(bucket) - 1);
-            sameKeyedBuckets.add(createBucket(key, bucket.docCount, bucket.aggregations));
+            sameKeyedBuckets.add(new Bucket(Math.round(key), bucket.docCount, format, bucket.aggregations));
         }
         if (sameKeyedBuckets.isEmpty() == false) {
             reduceContext.consumeBucketsAndMaybeBreak(1);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
index 96811ce424c..a14fca63154 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.rounding.DateTimeUnit;
-import org.elasticsearch.common.rounding.Rounding;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.ParsedMultiBucketAggregation;
@@ -51,14 +49,6 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati
     public void setUp() throws Exception {
         super.setUp();
         format = randomNumericDocValueFormat();
-
-        roundingInfos = new RoundingInfo[6];
-        roundingInfos[0] = new RoundingInfo(Rounding.builder(DateTimeUnit.SECOND_OF_MINUTE).build(), 1, 5, 10, 30);
-        roundingInfos[1] = new RoundingInfo(Rounding.builder(DateTimeUnit.MINUTES_OF_HOUR).build(), 1, 5, 10, 30);
-        roundingInfos[2] = new RoundingInfo(Rounding.builder(DateTimeUnit.HOUR_OF_DAY).build(), 1, 3, 12);
-        roundingInfos[3] = new RoundingInfo(Rounding.builder(DateTimeUnit.DAY_OF_MONTH).build(), 1, 7);
-        roundingInfos[4] = new RoundingInfo(Rounding.builder(DateTimeUnit.MONTH_OF_YEAR).build(), 1, 3);
-        roundingInfos[5] = new RoundingInfo(Rounding.builder(DateTimeUnit.YEAR_OF_CENTURY).build(), 1, 10, 20, 50, 100);
     }
 
     @Override
@@ -66,6 +56,8 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati
                                                        List<PipelineAggregator> pipelineAggregators,
                                                        Map<String, Object> metaData,
                                                        InternalAggregations aggregations) {
+        
+        roundingInfos = AutoDateHistogramAggregationBuilder.buildRoundings(null);
         int nbBuckets = randomNumberOfBuckets();
         int targetBuckets = randomIntBetween(1, nbBuckets * 2 + 1);
         List<InternalAutoDateHistogram.Bucket> buckets = new ArrayList<>(nbBuckets);
@@ -81,6 +73,7 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati
         InternalAggregations subAggregations = new InternalAggregations(Collections.emptyList());
         BucketInfo bucketInfo = new BucketInfo(roundingInfos, randomIntBetween(0, roundingInfos.length - 1), subAggregations);
 
+
         return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators, metaData);
     }
 
@@ -92,13 +85,50 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati
                 roundingIdx = histogram.getBucketInfo().roundingIdx;
             }
         }
-        Map<Long, Long> expectedCounts = new TreeMap<>();
-        for (Histogram histogram : inputs) {
+        RoundingInfo roundingInfo = roundingInfos[roundingIdx];
+
+        long lowest = Long.MAX_VALUE;
+        long highest = 0;
+        for (InternalAutoDateHistogram histogram : inputs) {
             for (Histogram.Bucket bucket : histogram.getBuckets()) {
-                expectedCounts.compute(roundingInfos[roundingIdx].rounding.round(((DateTime) bucket.getKey()).getMillis()),
-                        (key, oldValue) -> (oldValue == null ? 0 : oldValue) + bucket.getDocCount());
+                long bucketKey = ((DateTime) bucket.getKey()).getMillis();
+                if (bucketKey < lowest) {
+                    lowest = bucketKey;
+                }
+                if (bucketKey > highest) {
+                    highest = bucketKey;
+                }
             }
         }
+        long normalizedDuration = (highest - lowest) / roundingInfo.getRoughEstimateDurationMillis();
+        long innerIntervalToUse = 0;
+        for (int interval : roundingInfo.innerIntervals) {
+            if (normalizedDuration / interval < maxNumberOfBuckets()) {
+                innerIntervalToUse = interval;
+            }
+        }
+        Map<Long, Long> expectedCounts = new TreeMap<>();
+        long intervalInMillis = innerIntervalToUse*roundingInfo.getRoughEstimateDurationMillis();
+        for (long keyForBucket = roundingInfo.rounding.round(lowest);
+             keyForBucket <= highest;
+             keyForBucket = keyForBucket + intervalInMillis) {
+            expectedCounts.put(keyForBucket, 0L);
+
+            for (InternalAutoDateHistogram histogram : inputs) {
+                for (Histogram.Bucket bucket : histogram.getBuckets()) {
+                    long bucketKey = ((DateTime) bucket.getKey()).getMillis();
+                    long roundedBucketKey = roundingInfo.rounding.round(bucketKey);
+                    if (roundedBucketKey >= keyForBucket
+                        && roundedBucketKey < keyForBucket + intervalInMillis) {
+                        long count = bucket.getDocCount();
+                        expectedCounts.compute(keyForBucket,
+                            (key, oldValue) -> (oldValue == null ? 0 : oldValue) + count);
+                    }
+                }
+            }
+        }
+
+
         Map<Long, Long> actualCounts = new TreeMap<>();
         for (Histogram.Bucket bucket : reduced.getBuckets()) {
             actualCounts.compute(((DateTime) bucket.getKey()).getMillis(),
@@ -117,12 +147,6 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati
         return ParsedAutoDateHistogram.class;
     }
 
-    @Override
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32215")
-    public void testReduceRandom() {
-        super.testReduceRandom();
-    }
-
     @Override
     protected InternalAutoDateHistogram mutateInstance(InternalAutoDateHistogram instance) {
         String name = instance.getName();

From adb93da974f88d7939eeac2fbe9066d4555ca586 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel@mitterdorfer.name>
Date: Tue, 31 Jul 2018 15:04:07 +0200
Subject: [PATCH 240/260] Mute KerberosAuthenticationIT

Relates #32498
---
 .../security/authc/kerberos/KerberosAuthenticationIT.java   | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
index d5928cb58f6..a36040d1ba8 100644
--- a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
+++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
@@ -82,6 +82,7 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         assertOK(response);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32498")
     public void testLoginByKeytab() throws IOException, PrivilegedActionException {
         final String userPrincipalName = System.getProperty(TEST_USER_WITH_KEYTAB_KEY);
         final String keytabPath = System.getProperty(TEST_USER_WITH_KEYTAB_PATH_KEY);
@@ -91,6 +92,7 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         executeRequestAndVerifyResponse(userPrincipalName, callbackHandler);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32498")
     public void testLoginByUsernamePassword() throws IOException, PrivilegedActionException {
         final String userPrincipalName = System.getProperty(TEST_USER_WITH_PWD_KEY);
         final String password = System.getProperty(TEST_USER_WITH_PWD_PASSWD_KEY);
@@ -100,6 +102,10 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         executeRequestAndVerifyResponse(userPrincipalName, callbackHandler);
     }
 
+    public void testSoDoesNotFailWithNoTests() {
+        // intentionally empty - this is just needed to ensure the build does not fail because we mute its only test.
+    }
+
     private void executeRequestAndVerifyResponse(final String userPrincipalName,
             final SpnegoHttpClientConfigCallbackHandler callbackHandler) throws PrivilegedActionException, IOException {
         final Request request = new Request("GET", "/_xpack/security/_authenticate");

From 4fa92cbf49326b45509ce2593f962f276282dca9 Mon Sep 17 00:00:00 2001
From: Sohaib Iftikhar <sohaib1692@gmail.com>
Date: Tue, 31 Jul 2018 16:11:17 +0200
Subject: [PATCH 241/260] Changed ReindexRequest to use Writeable.Reader
 (#32401)

-- This is a pre-stage for adding the reindex API to the REST high-level-client
-- Follows the pattern set in #26315
---
 .../reindex/TransportDeleteByQueryAction.java |  4 +--
 .../index/reindex/TransportReindexAction.java |  3 +-
 .../reindex/TransportUpdateByQueryAction.java |  4 +--
 .../index/reindex/RoundTripTests.java         | 35 ++++++++-----------
 .../index/reindex/DeleteByQueryRequest.java   |  7 ++++
 .../index/reindex/ReindexRequest.java         | 12 ++++---
 .../index/reindex/UpdateByQueryRequest.java   |  8 +++--
 7 files changed, 42 insertions(+), 31 deletions(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
index c1defe56adc..706f2c0b8f8 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
@@ -27,13 +27,13 @@ import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.function.Supplier;
 
 public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, BulkByScrollResponse> {
 
@@ -46,7 +46,7 @@ public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteB
     public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, Client client,
                                         TransportService transportService, ScriptService scriptService, ClusterService clusterService) {
         super(settings, DeleteByQueryAction.NAME, transportService, actionFilters,
-            (Supplier<DeleteByQueryRequest>) DeleteByQueryRequest::new);
+            (Writeable.Reader<DeleteByQueryRequest>) DeleteByQueryRequest::new);
         this.threadPool = threadPool;
         this.client = client;
         this.scriptService = scriptService;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index e54b5f50ae6..1d80c28b585 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -39,6 +39,7 @@ import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.action.index.IndexRequest;
@@ -104,7 +105,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
     public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
             IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, ScriptService scriptService,
             AutoCreateIndex autoCreateIndex, Client client, TransportService transportService) {
-        super(settings, ReindexAction.NAME, transportService, actionFilters, ReindexRequest::new);
+        super(settings, ReindexAction.NAME, transportService, actionFilters, (Writeable.Reader<ReindexRequest>)ReindexRequest::new);
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.scriptService = scriptService;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index 34ae3fdd0c6..00d14822ba0 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -29,6 +29,7 @@ import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
@@ -43,7 +44,6 @@ import org.elasticsearch.transport.TransportService;
 
 import java.util.Map;
 import java.util.function.BiFunction;
-import java.util.function.Supplier;
 
 public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkByScrollResponse> {
 
@@ -56,7 +56,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
     public TransportUpdateByQueryAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, Client client,
                                         TransportService transportService, ScriptService scriptService, ClusterService clusterService) {
         super(settings, UpdateByQueryAction.NAME, transportService, actionFilters,
-            (Supplier<UpdateByQueryRequest>) UpdateByQueryRequest::new);
+            (Writeable.Reader<UpdateByQueryRequest>) UpdateByQueryRequest::new);
         this.threadPool = threadPool;
         this.client = client;
         this.scriptService = scriptService;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index 2dc4b59e8d9..97809c9bc8d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -67,19 +67,17 @@ public class RoundTripTests extends ESTestCase {
                 new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), port, null,
                     query, username, password, headers, socketTimeout, connectTimeout));
         }
-        ReindexRequest tripped = new ReindexRequest();
-        roundTrip(reindex, tripped);
+        ReindexRequest tripped = new ReindexRequest(toInputByteStream(reindex));
         assertRequestEquals(reindex, tripped);
 
         // Try slices=auto with a version that doesn't support it, which should fail
         reindex.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
-        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, reindex, null));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> toInputByteStream(Version.V_6_0_0_alpha1, reindex));
         assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage());
 
         // Try regular slices with a version that doesn't support slices=auto, which should succeed
-        tripped = new ReindexRequest();
         reindex.setSlices(between(1, Integer.MAX_VALUE));
-        roundTrip(Version.V_6_0_0_alpha1, reindex, tripped);
+        tripped = new ReindexRequest(toInputByteStream(reindex));
         assertRequestEquals(Version.V_6_0_0_alpha1, reindex, tripped);
     }
 
@@ -89,20 +87,18 @@ public class RoundTripTests extends ESTestCase {
         if (randomBoolean()) {
             update.setPipeline(randomAlphaOfLength(5));
         }
-        UpdateByQueryRequest tripped = new UpdateByQueryRequest();
-        roundTrip(update, tripped);
+        UpdateByQueryRequest tripped = new UpdateByQueryRequest(toInputByteStream(update));
         assertRequestEquals(update, tripped);
         assertEquals(update.getPipeline(), tripped.getPipeline());
 
         // Try slices=auto with a version that doesn't support it, which should fail
         update.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
-        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, update, null));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> toInputByteStream(Version.V_6_0_0_alpha1, update));
         assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage());
 
         // Try regular slices with a version that doesn't support slices=auto, which should succeed
-        tripped = new UpdateByQueryRequest();
         update.setSlices(between(1, Integer.MAX_VALUE));
-        roundTrip(Version.V_6_0_0_alpha1, update, tripped);
+        tripped = new UpdateByQueryRequest(toInputByteStream(update));
         assertRequestEquals(update, tripped);
         assertEquals(update.getPipeline(), tripped.getPipeline());
     }
@@ -110,19 +106,17 @@ public class RoundTripTests extends ESTestCase {
     public void testDeleteByQueryRequest() throws IOException {
         DeleteByQueryRequest delete = new DeleteByQueryRequest(new SearchRequest());
         randomRequest(delete);
-        DeleteByQueryRequest tripped = new DeleteByQueryRequest();
-        roundTrip(delete, tripped);
+        DeleteByQueryRequest tripped = new DeleteByQueryRequest(toInputByteStream(delete));
         assertRequestEquals(delete, tripped);
 
         // Try slices=auto with a version that doesn't support it, which should fail
         delete.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
-        Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, delete, null));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> toInputByteStream(Version.V_6_0_0_alpha1, delete));
         assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage());
 
         // Try regular slices with a version that doesn't support slices=auto, which should succeed
-        tripped = new DeleteByQueryRequest();
         delete.setSlices(between(1, Integer.MAX_VALUE));
-        roundTrip(Version.V_6_0_0_alpha1, delete, tripped);
+        tripped = new DeleteByQueryRequest(toInputByteStream(delete));
         assertRequestEquals(delete, tripped);
     }
 
@@ -198,23 +192,24 @@ public class RoundTripTests extends ESTestCase {
             request.setTaskId(new TaskId(randomAlphaOfLength(5), randomLong()));
         }
         RethrottleRequest tripped = new RethrottleRequest();
-        roundTrip(request, tripped);
+        // We use readFrom here because Rethrottle does not support the Writeable.Reader interface
+        tripped.readFrom(toInputByteStream(request));
         assertEquals(request.getRequestsPerSecond(), tripped.getRequestsPerSecond(), 0.00001);
         assertArrayEquals(request.getActions(), tripped.getActions());
         assertEquals(request.getTaskId(), tripped.getTaskId());
     }
 
-    private void roundTrip(Streamable example, Streamable empty) throws IOException {
-        roundTrip(Version.CURRENT, example, empty);
+    private StreamInput toInputByteStream(Streamable example) throws IOException {
+        return toInputByteStream(Version.CURRENT, example);
     }
 
-    private void roundTrip(Version version, Streamable example, Streamable empty) throws IOException {
+    private StreamInput toInputByteStream(Version version, Streamable example) throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         out.setVersion(version);
         example.writeTo(out);
         StreamInput in = out.bytes().streamInput();
         in.setVersion(version);
-        empty.readFrom(in);
+        return in;
     }
 
     private Script randomScript() {
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
index aa8543175d9..f848e8722c7 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
@@ -23,8 +23,11 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.tasks.TaskId;
 
+import java.io.IOException;
+
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
@@ -53,6 +56,10 @@ public class DeleteByQueryRequest extends AbstractBulkByScrollRequest<DeleteByQu
         this(search, true);
     }
 
+    public DeleteByQueryRequest(StreamInput in) throws IOException {
+        super.readFrom(in);
+    }
+
     private DeleteByQueryRequest(SearchRequest search, boolean setDefaults) {
         super(search, setDefaults);
         // Delete-By-Query does not require the source
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index 276c4559153..e45d039edae 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -59,6 +59,13 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
         this.destination = destination;
     }
 
+    public ReindexRequest(StreamInput in) throws IOException {
+        super.readFrom(in);
+        destination = new IndexRequest();
+        destination.readFrom(in);
+        remoteInfo = in.readOptionalWriteable(RemoteInfo::new);
+    }
+
     @Override
     protected ReindexRequest self() {
         return this;
@@ -135,10 +142,7 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        destination = new IndexRequest();
-        destination.readFrom(in);
-        remoteInfo = in.readOptionalWriteable(RemoteInfo::new);
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
index 2be0e46dbce..eb4fd59a7bc 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
@@ -47,6 +47,11 @@ public class UpdateByQueryRequest extends AbstractBulkIndexByScrollRequest<Updat
         this(search, true);
     }
 
+    public UpdateByQueryRequest(StreamInput in) throws IOException {
+        super.readFrom(in);
+        pipeline = in.readOptionalString();
+    }
+
     private UpdateByQueryRequest(SearchRequest search, boolean setDefaults) {
         super(search, setDefaults);
     }
@@ -108,8 +113,7 @@ public class UpdateByQueryRequest extends AbstractBulkIndexByScrollRequest<Updat
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        pipeline = in.readOptionalString();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override

From 736d2e8a65bf314286b53223753b6749e785725c Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Tue, 31 Jul 2018 16:14:37 +0200
Subject: [PATCH 242/260] Docs: Fix README upgrade mention (#32313)

The README still mentioned Elasticsearch 1.x. This commit links to the
documentation instead and properly formats the last paragraph.
---
 README.textile | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/README.textile b/README.textile
index ce7b3b7d344..aa61c035742 100644
--- a/README.textile
+++ b/README.textile
@@ -209,10 +209,6 @@ The distribution for each project will be created under the @build/distributions
 
 See the "TESTING":TESTING.asciidoc file for more information about running the Elasticsearch test suite.
 
-h3. Upgrading from Elasticsearch 1.x?
+h3. Upgrading from older Elasticsearch versions
 
-In order to ensure a smooth upgrade process from earlier versions of
-Elasticsearch (1.x), it is required to perform a full cluster restart. Please
-see the "setup reference":
-https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html
-for more details on the upgrade process.
+In order to ensure a smooth upgrade process from earlier versions of Elasticsearch, please see our "upgrade documentation":https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html for more details on the upgrade process.

From b42765c82fa561506f12c0363b2e30e73d352497 Mon Sep 17 00:00:00 2001
From: Vladimir Dolzhenko <vladimir.dolzhenko@gmail.com>
Date: Tue, 31 Jul 2018 16:21:48 +0200
Subject: [PATCH 243/260] Increase max chunk size to 256Mb for repo-azure
 (#32101)

Increase max chunk size to 256Mb for repo-azure
---
 plugins/repository-azure/build.gradle         |  17 +-
 .../azure-keyvault-core-0.8.0.jar.sha1        |   1 -
 .../azure-keyvault-core-1.0.0.jar.sha1        |   1 +
 .../licenses/azure-storage-5.0.0.jar.sha1     |   1 -
 .../licenses/azure-storage-8.0.0.jar.sha1     |   1 +
 .../licenses/guava-20.0.jar.sha1              |   1 +
 .../licenses/guava-LICENSE.txt                | 202 ++++++++++++++++++
 .../licenses/guava-NOTICE.txt                 |   0
 .../azure/AzureStorageService.java            |   5 +-
 .../azure/AzureRepositorySettingsTests.java   |   6 +-
 10 files changed, 227 insertions(+), 8 deletions(-)
 delete mode 100644 plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/azure-keyvault-core-1.0.0.jar.sha1
 delete mode 100644 plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/azure-storage-8.0.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/guava-20.0.jar.sha1
 create mode 100644 plugins/repository-azure/licenses/guava-LICENSE.txt
 create mode 100644 plugins/repository-azure/licenses/guava-NOTICE.txt

diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index 6d7412f8250..13ec8031c0f 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -23,8 +23,9 @@ esplugin {
 }
 
 dependencies {
-  compile 'com.microsoft.azure:azure-storage:5.0.0'
-  compile 'com.microsoft.azure:azure-keyvault-core:0.8.0'
+  compile 'com.microsoft.azure:azure-storage:8.0.0'
+  compile 'com.microsoft.azure:azure-keyvault-core:1.0.0'
+  compile 'com.google.guava:guava:20.0'
   compile 'org.apache.commons:commons-lang3:3.4'
 }
 
@@ -40,6 +41,18 @@ thirdPartyAudit.excludes = [
   // Optional and not enabled by Elasticsearch
   'org.slf4j.Logger',
   'org.slf4j.LoggerFactory',
+  // uses internal java api: sun.misc.Unsafe
+  'com.google.common.cache.Striped64',
+  'com.google.common.cache.Striped64$1',
+  'com.google.common.cache.Striped64$Cell',
+  'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+  'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+  'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
+  'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+  'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+  'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+  'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+  'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
 ]
 
 check {
diff --git a/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1 b/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1
deleted file mode 100644
index b86c58db842..00000000000
--- a/plugins/repository-azure/licenses/azure-keyvault-core-0.8.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35f7ac687462f491d0f8b0d96733dfe347493d70
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-keyvault-core-1.0.0.jar.sha1 b/plugins/repository-azure/licenses/azure-keyvault-core-1.0.0.jar.sha1
new file mode 100644
index 00000000000..8b44e99aee1
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-keyvault-core-1.0.0.jar.sha1
@@ -0,0 +1 @@
+e89dd5e621e21b753096ec6a03f203c01482c612
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1
deleted file mode 100644
index 9882cb80204..00000000000
--- a/plugins/repository-azure/licenses/azure-storage-5.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ba8f04bfeac08016c0f88423a202d0f3aac03aed
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/azure-storage-8.0.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-8.0.0.jar.sha1
new file mode 100644
index 00000000000..4e333ad8241
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-storage-8.0.0.jar.sha1
@@ -0,0 +1 @@
+f6759c16ade4e2a05bc1dfbaf55161b9ed0e78b9
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/guava-20.0.jar.sha1 b/plugins/repository-azure/licenses/guava-20.0.jar.sha1
new file mode 100644
index 00000000000..7b6ae09060b
--- /dev/null
+++ b/plugins/repository-azure/licenses/guava-20.0.jar.sha1
@@ -0,0 +1 @@
+89507701249388e1ed5ddcf8c41f4ce1be7831ef
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/guava-LICENSE.txt b/plugins/repository-azure/licenses/guava-LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/plugins/repository-azure/licenses/guava-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/guava-NOTICE.txt b/plugins/repository-azure/licenses/guava-NOTICE.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
index 9482182b02d..75bcc3e9c1e 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
@@ -61,7 +61,10 @@ import static java.util.Collections.emptyMap;
 public class AzureStorageService extends AbstractComponent {
 
     public static final ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
-    public static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
+    /**
+     * {@link com.microsoft.azure.storage.blob.BlobConstants#MAX_SINGLE_UPLOAD_BLOB_SIZE_IN_BYTES}
+     */
+    public static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(256, ByteSizeUnit.MB);
 
     // 'package' for testing
     volatile Map<String, AzureStorageSettings> storageSettings = emptyMap();
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
index b4b71577cbc..43891a8e9d5 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
@@ -104,7 +104,7 @@ public class AzureRepositorySettingsTests extends ESTestCase {
         assertEquals(AzureStorageService.MAX_CHUNK_SIZE, azureRepository.chunkSize());
 
         // chunk size in settings
-        int size = randomIntBetween(1, 64);
+        int size = randomIntBetween(1, 256);
         azureRepository = azureRepository(Settings.builder().put("chunk_size", size + "mb").build());
         assertEquals(new ByteSizeValue(size, ByteSizeUnit.MB), azureRepository.chunkSize());
 
@@ -120,8 +120,8 @@ public class AzureRepositorySettingsTests extends ESTestCase {
 
         // greater than max chunk size not allowed
         e = expectThrows(IllegalArgumentException.class, () ->
-            azureRepository(Settings.builder().put("chunk_size", "65mb").build()));
-        assertEquals("failed to parse value [65mb] for setting [chunk_size], must be <= [64mb]", e.getMessage());
+            azureRepository(Settings.builder().put("chunk_size", "257mb").build()));
+        assertEquals("failed to parse value [257mb] for setting [chunk_size], must be <= [256mb]", e.getMessage());
     }
 
 }

From 5f302580f9dea18b389007b861fc06b013e94499 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel@mitterdorfer.name>
Date: Tue, 31 Jul 2018 16:17:28 +0200
Subject: [PATCH 244/260] Mute SSLTrustRestrictionsTests on JDK 11

This commit adds an assumption to two test methods in
SSLTrustRestrictionsTests that we are not on JDK 11 as the tests
currently fail there.

Relates #29989
---
 .../elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
index cf77ca975a4..e1896e01365 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ssl;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -173,6 +174,8 @@ public class SSLTrustRestrictionsTests extends SecurityIntegTestCase {
     }
 
     public void testCertificateWithUntrustedNameFails() throws Exception {
+        // see https://github.com/elastic/elasticsearch/issues/29989
+        assumeTrue("test fails on JDK 11 currently", JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0);
         writeRestrictions("*.trusted");
         try {
             tryConnect(untrustedCert);
@@ -183,6 +186,8 @@ public class SSLTrustRestrictionsTests extends SecurityIntegTestCase {
     }
 
     public void testRestrictionsAreReloaded() throws Exception {
+        // see https://github.com/elastic/elasticsearch/issues/29989
+        assumeTrue("test fails on JDK 11 currently", JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0);
         writeRestrictions("*");
         assertBusy(() -> {
             try {

From 22459576d757833d9dfbca59cb44b09fc8bd77b1 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 31 Jul 2018 10:54:24 -0400
Subject: [PATCH 245/260] Logging: Make node name consistent in logger (#31588)

First, some background: we have 15 different methods to get a logger in
Elasticsearch but they can be broken down into three broad categories
based on what information is provided when building the logger.

Just a class like:
```
private static final Logger logger = ESLoggerFactory.getLogger(ActionModule.class);
```
or:
```
protected final Logger logger = Loggers.getLogger(getClass());
```

The class and settings:
```
this.logger = Loggers.getLogger(getClass(), settings);
```

Or more information like:
```
Loggers.getLogger("index.store.deletes", settings, shardId)
```

The goal of the "class and settings" variant is to attach the node name
to the logger. Because we don't always have the settings available, we
often use the "just a class" variant and get loggers without node names
attached. There isn't any real consistency here. Some loggers get the
node name because it is convenient and some do not.

This change makes the node name available to all loggers all the time.
Almost. There are some caveats are testing that I'll get to. But in
*production* code the node name is node available to all loggers. This
means we can stop using the "class and settings" variants to fetch
loggers which was the real goal here, but a pleasant side effect is that
the ndoe name is now consitent on every log line and optional by editing
the logging pattern. This is all powered by setting the node name
statically on a logging formatter very early in initialization.

Now to tests: tests can't set the node name statically because
subclasses of `ESIntegTestCase` run many nodes in the same jvm, even in
the same class loader. Also, lots of tests don't run with a real node so
they don't *have* a node name at all. To support multiple nodes in the
same JVM tests suss out the node name from the thread name which works
surprisingly well and easy to test in a nice way. For those threads
that are not part of an `ESIntegTestCase` node we stick whatever useful
information we can get form the thread name in the place of the node
name. This allows us to keep the logger format consistent.
---
 .../src/main/resources/log4j2.properties      |   2 +-
 .../src/main/resources/log4j2.properties      |   2 +-
 distribution/src/config/log4j2.properties     |  10 +-
 docs/reference/index-modules/slowlog.asciidoc |   4 +-
 docs/reference/setup/logging-config.asciidoc  |   2 +-
 .../bootstrap/EvilBootstrapChecksTests.java   |   2 +-
 .../common/logging/EvilLoggerTests.java       |  25 +++-
 .../common/logging/config/log4j2.properties   |   7 +-
 .../logging/config/third/log4j2.properties    |   2 +-
 .../logging/deprecation/log4j2.properties     |   6 +-
 .../logging/find_appender/log4j2.properties   |   2 +-
 .../logging/location_info/log4j2.properties   |   4 +-
 .../logging/no_node_name/log4j2.properties    |  25 ++++
 .../common/logging/prefix/log4j2.properties   |   4 +-
 .../common/logging/settings/log4j2.properties |   6 +-
 .../elasticsearch/bootstrap/Bootstrap.java    |   8 +-
 .../bootstrap/BootstrapChecks.java            |  10 +-
 ...ElasticsearchUncaughtExceptionHandler.java |  11 +-
 .../common/logging/ESLoggerFactory.java       |   9 ++
 .../common/logging/LogConfigurator.java       |  73 +++++++++++-
 .../elasticsearch/common/logging/Loggers.java |  29 ++---
 .../logging/NodeNamePatternConverter.java     |  72 ++++++++++++
 .../java/org/elasticsearch/node/Node.java     |  17 ++-
 .../bootstrap/BootstrapChecksTests.java       |  89 +++++++-------
 ...icsearchUncaughtExceptionHandlerTests.java |   4 +-
 .../index/engine/InternalEngineTests.java     |   2 +-
 .../TestThreadInfoPatternConverter.java       | 111 ++++++++++++++++++
 .../org/elasticsearch/test/ESTestCase.java    |  16 ++-
 .../src/main/resources/log4j2-test.properties |   2 +-
 .../resources/log4j2.component.properties     |   9 ++
 .../TestThreadInfoPatternConverterTests.java  |  53 +++++++++
 .../esnative/ESNativeRealmMigrateTool.java    |  11 +-
 32 files changed, 493 insertions(+), 136 deletions(-)
 create mode 100644 qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties
 create mode 100644 server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java
 create mode 100644 test/framework/src/main/resources/log4j2.component.properties
 create mode 100644 test/framework/src/test/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverterTests.java

diff --git a/benchmarks/src/main/resources/log4j2.properties b/benchmarks/src/main/resources/log4j2.properties
index c3ae1fe56d3..808d611670f 100644
--- a/benchmarks/src/main/resources/log4j2.properties
+++ b/benchmarks/src/main/resources/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %m%n
 
 # Do not log at all if it is not really critical - we're in a benchmark
 rootLogger.level = error
diff --git a/client/benchmark/src/main/resources/log4j2.properties b/client/benchmark/src/main/resources/log4j2.properties
index 8652131bf49..a68c7f5653f 100644
--- a/client/benchmark/src/main/resources/log4j2.properties
+++ b/client/benchmark/src/main/resources/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties
index 589f529e83e..6de21cd48f6 100644
--- a/distribution/src/config/log4j2.properties
+++ b/distribution/src/config/log4j2.properties
@@ -7,13 +7,13 @@ logger.action.level = debug
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
 
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
 appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n
 appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
@@ -38,7 +38,7 @@ appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
 appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_rolling.layout.type = PatternLayout
-appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n
 appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
 appender.deprecation_rolling.policies.type = Policies
 appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
@@ -55,7 +55,7 @@ appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%i.log.gz
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
@@ -72,7 +72,7 @@ appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
 appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n
 appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%i.log.gz
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc
index 1dc84f380f5..235256bdce7 100644
--- a/docs/reference/index-modules/slowlog.asciidoc
+++ b/docs/reference/index-modules/slowlog.asciidoc
@@ -50,7 +50,7 @@ appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%i.log.gz
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
@@ -103,7 +103,7 @@ appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
 appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%node_name]%marker %.-10000m%n
 appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%i.log.gz
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy
diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc
index d35b3db2df7..ac949bc941a 100644
--- a/docs/reference/setup/logging-config.asciidoc
+++ b/docs/reference/setup/logging-config.asciidoc
@@ -25,7 +25,7 @@ appender.rolling.type = RollingFile <1>
 appender.rolling.name = rolling
 appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log <2>
 appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %.-10000m%n
 appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz <3>
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy <4>
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java
index 0dc9ea0a170..b034cd08d77 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java
@@ -87,7 +87,7 @@ public class EvilBootstrapChecksTests extends ESTestCase {
         final boolean enforceLimits = randomBoolean();
         final IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
-                () -> BootstrapChecks.check(new BootstrapContext(Settings.EMPTY, null), enforceLimits, emptyList(), "testInvalidValue"));
+                () -> BootstrapChecks.check(new BootstrapContext(Settings.EMPTY, null), enforceLimits, emptyList()));
         final Matcher<String> matcher = containsString(
                 "[es.enforce.bootstrap.checks] must be [true] but was [" + value + "]");
         assertThat(e, hasToString(matcher));
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
index d4bc754689e..9285abad9b9 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
@@ -56,6 +56,7 @@ import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.lessThan;
@@ -319,9 +320,11 @@ public class EvilLoggerTests extends ESTestCase {
         assertThat(events.size(), equalTo(expectedLogLines + stackTraceLength));
         for (int i = 0; i < expectedLogLines; i++) {
             if (prefix == null) {
-                assertThat(events.get(i), startsWith("test"));
+                assertThat("Contents of [" + path + "] are wrong",
+                        events.get(i), startsWith("[" + getTestName() + "] test"));
             } else {
-                assertThat(events.get(i), startsWith("[" + prefix + "] test"));
+                assertThat("Contents of [" + path + "] are wrong",
+                        events.get(i), startsWith("[" + getTestName() + "][" + prefix + "] test"));
             }
         }
     }
@@ -357,6 +360,24 @@ public class EvilLoggerTests extends ESTestCase {
         }
     }
 
+    public void testNoNodeNameWarning() throws IOException, UserException {
+        setupLogging("no_node_name");
+
+        final String path =
+            System.getProperty("es.logs.base_path") +
+                System.getProperty("file.separator") +
+                System.getProperty("es.logs.cluster_name") + ".log";
+        final List<String> events = Files.readAllLines(PathUtils.get(path));
+        assertThat(events.size(), equalTo(2));
+        final String location = "org.elasticsearch.common.logging.LogConfigurator";
+        // the first message is a warning for unsupported configuration files
+        assertLogLine(events.get(0), Level.WARN, location, "\\[null\\] Some logging configurations have %marker but don't "
+                + "have %node_name. We will automatically add %node_name to the pattern to ease the migration for users "
+                + "who customize log4j2.properties but will stop this behavior in 7.0. You should manually replace "
+                + "`%node_name` with `\\[%node_name\\]%marker ` in these locations:");
+        assertThat(events.get(1), endsWith("no_node_name/log4j2.properties"));
+    }
+
     private void setupLogging(final String config) throws IOException, UserException {
         setupLogging(config, Settings.EMPTY);
     }
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
index 8b956421458..2c9f48a359a 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %marker%m%n
+appender.file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
@@ -23,10 +23,9 @@ appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
 appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
-appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+appender.deprecation_file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 logger.deprecation.name = deprecation
 logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
 logger.deprecation.additivity = false
-
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
index ed794cb7c3b..8a9ec160fb1 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/third/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console3.type = Console
 appender.console3.name = console3
 appender.console3.layout.type = PatternLayout
-appender.console3.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console3.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 logger.third.name = third
 logger.third.level = debug
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
index fd7af2ce731..388c9f9b2fc 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %marker%m%n
+appender.file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
@@ -17,7 +17,7 @@ appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
 appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
-appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+appender.deprecation_file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 logger.deprecation.name = deprecation
 logger.deprecation.level = warn
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties
index 8553ec5e791..d036e571cf0 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/find_appender/log4j2.properties
@@ -1,7 +1,7 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 logger.has_console_appender.name = has_console_appender
 logger.has_console_appender.level = trace
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
index 5fa85b5d156..ee59755a2d7 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %marker%m%n
+appender.file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties
new file mode 100644
index 00000000000..fd7af2ce731
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties
@@ -0,0 +1,25 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = [%p][%l] %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
+
+appender.deprecation_file.type = File
+appender.deprecation_file.name = deprecation_file
+appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+appender.deprecation_file.layout.type = PatternLayout
+appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+
+logger.deprecation.name = deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
+logger.deprecation.additivity = false
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
index 5dfa369a3bd..199fddc31c6 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %marker%m%n
+appender.file.layout.pattern = [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties
index 1acb65511d6..abe4a279dc8 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties
@@ -1,13 +1,13 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] [%test_thread_info]%marker %m%n
 
 appender.file.type = File
 appender.file.name = file
 appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = [%p][%l] %marker%m%n
+appender.file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
@@ -17,7 +17,7 @@ appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
 appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
-appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
+appender.deprecation_file.layout.pattern = [%p][%l] [%test_thread_info]%marker %m%n
 
 logger.deprecation.name = org.elasticsearch.deprecation.common.settings
 logger.deprecation.level = warn
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 870c537e020..76db6db7674 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -37,6 +37,7 @@ import org.elasticsearch.common.inject.CreationException;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.logging.NodeNamePatternConverter;
 import org.elasticsearch.common.network.IfConfig;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureSettings;
@@ -287,6 +288,10 @@ final class Bootstrap {
 
         final SecureSettings keystore = loadSecureSettings(initialEnv);
         final Environment environment = createEnvironment(pidFile, keystore, initialEnv.settings(), initialEnv.configFile());
+
+        String nodeName = Node.NODE_NAME_SETTING.get(environment.settings());
+        NodeNamePatternConverter.setNodeName(nodeName);
+
         try {
             LogConfigurator.configure(environment);
         } catch (IOException e) {
@@ -317,8 +322,7 @@ final class Bootstrap {
             // install the default uncaught exception handler; must be done before security is
             // initialized as we do not want to grant the runtime permission
             // setDefaultUncaughtExceptionHandler
-            Thread.setDefaultUncaughtExceptionHandler(
-                new ElasticsearchUncaughtExceptionHandler(() -> Node.NODE_NAME_SETTING.get(environment.settings())));
+            Thread.setDefaultUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler());
 
             INSTANCE.setup(true, environment);
 
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
index 19fdb8837d6..1a028042db2 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -30,7 +30,6 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.monitor.process.ProcessProbe;
-import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
 
 import java.io.BufferedReader;
@@ -74,8 +73,7 @@ final class BootstrapChecks {
         combinedChecks.addAll(additionalChecks);
         check(  context,
                 enforceLimits(boundTransportAddress, DiscoveryModule.DISCOVERY_TYPE_SETTING.get(context.settings)),
-                Collections.unmodifiableList(combinedChecks),
-                Node.NODE_NAME_SETTING.get(context.settings));
+                Collections.unmodifiableList(combinedChecks));
     }
 
     /**
@@ -86,14 +84,12 @@ final class BootstrapChecks {
      * @param context        the current node boostrap context
      * @param enforceLimits {@code true} if the checks should be enforced or otherwise warned
      * @param checks        the checks to execute
-     * @param nodeName      the node name to be used as a logging prefix
      */
     static void check(
         final BootstrapContext context,
         final boolean enforceLimits,
-        final List<BootstrapCheck> checks,
-        final String nodeName) throws NodeValidationException {
-        check(context, enforceLimits, checks, Loggers.getLogger(BootstrapChecks.class, nodeName));
+        final List<BootstrapCheck> checks) throws NodeValidationException {
+        check(context, enforceLimits, checks, Loggers.getLogger(BootstrapChecks.class));
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
index 857ff65b6c2..1ef9b7740c2 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
@@ -27,16 +27,9 @@ import org.elasticsearch.common.logging.Loggers;
 import java.io.IOError;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
-import java.util.Objects;
-import java.util.function.Supplier;
 
 class ElasticsearchUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
-
-    private final Supplier<String> loggingPrefixSupplier;
-
-    ElasticsearchUncaughtExceptionHandler(final Supplier<String> loggingPrefixSupplier) {
-        this.loggingPrefixSupplier = Objects.requireNonNull(loggingPrefixSupplier);
-    }
+    private static final Logger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class);
 
     @Override
     public void uncaughtException(Thread t, Throwable e) {
@@ -70,12 +63,10 @@ class ElasticsearchUncaughtExceptionHandler implements Thread.UncaughtExceptionH
     }
 
     void onFatalUncaught(final String threadName, final Throwable t) {
-        final Logger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class, loggingPrefixSupplier.get());
         logger.error(() -> new ParameterizedMessage("fatal error in thread [{}], exiting", threadName), t);
     }
 
     void onNonFatalUncaught(final String threadName, final Throwable t) {
-        final Logger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class, loggingPrefixSupplier.get());
         logger.warn(() -> new ParameterizedMessage("uncaught exception in thread [{}]", threadName), t);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java b/server/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
index 44d7d17b593..caa922b92cf 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
@@ -46,6 +46,15 @@ public final class ESLoggerFactory {
     }
 
     public static Logger getLogger(String prefix, Logger logger) {
+        /*
+         * In a followup we'll throw an exception if prefix is null or empty
+         * redirecting folks to LogManager.getLogger.
+         *
+         * This and more is tracked in https://github.com/elastic/elasticsearch/issues/32174
+         */
+        if (prefix == null || prefix.length() == 0) {
+            return logger;
+        }
         return new PrefixLogger((ExtendedLogger)logger, logger.getName(), prefix);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index 01aca53db05..4e3771f3668 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -23,12 +23,16 @@ import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.config.AbstractConfiguration;
+import org.apache.logging.log4j.core.config.ConfigurationException;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
 import org.apache.logging.log4j.core.config.Configurator;
 import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
 import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory;
 import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
 import org.apache.logging.log4j.core.config.composite.CompositeConfiguration;
+import org.apache.logging.log4j.core.config.plugins.util.PluginManager;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfiguration;
+import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationBuilder;
 import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationFactory;
 import org.apache.logging.log4j.status.StatusConsoleListener;
 import org.apache.logging.log4j.status.StatusData;
@@ -43,6 +47,7 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.FileVisitOption;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
@@ -50,9 +55,12 @@ import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.StreamSupport;
@@ -119,6 +127,13 @@ public class LogConfigurator {
         configure(environment.settings(), environment.configFile(), environment.logsFile());
     }
 
+    /**
+     * Load logging plugins so we can have {@code node_name} in the pattern.
+     */
+    public static void loadLog4jPlugins() {
+        PluginManager.addPackage(LogConfigurator.class.getPackage().getName());
+    }
+
     private static void checkErrorListener() {
         assert errorListenerIsRegistered() : "expected error listener to be registered";
         if (error.get()) {
@@ -135,6 +150,8 @@ public class LogConfigurator {
         Objects.requireNonNull(configsPath);
         Objects.requireNonNull(logsPath);
 
+        loadLog4jPlugins();
+
         setLogConfigurationSystemProperty(logsPath, settings);
         // we initialize the status logger immediately otherwise Log4j will complain when we try to get the context
         configureStatusLogger();
@@ -142,7 +159,53 @@ public class LogConfigurator {
         final LoggerContext context = (LoggerContext) LogManager.getContext(false);
 
         final List<AbstractConfiguration> configurations = new ArrayList<>();
-        final PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory();
+
+        /*
+         * Subclass the properties configurator to hack the new pattern in
+         * place so users don't have to change log4j2.properties in
+         * a minor release. In 7.0 we'll remove this and force users to
+         * change log4j2.properties. If they don't customize log4j2.properties
+         * then they won't have to do anything anyway.
+         *
+         * Everything in this subclass that isn't marked as a hack is copied
+         * from log4j2's source.
+         */
+        Set<String> locationsWithDeprecatedPatterns = Collections.synchronizedSet(new HashSet<>());
+        final PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory() {
+            @Override
+            public PropertiesConfiguration getConfiguration(final LoggerContext loggerContext, final ConfigurationSource source) {
+                final Properties properties = new Properties();
+                try (InputStream configStream = source.getInputStream()) {
+                    properties.load(configStream);
+                } catch (final IOException ioe) {
+                    throw new ConfigurationException("Unable to load " + source.toString(), ioe);
+                }
+                // Hack the new pattern into place
+                for (String name : properties.stringPropertyNames()) {
+                    if (false == name.endsWith(".pattern")) continue;
+                    // Null is weird here but we can't do anything with it so ignore it
+                    String value = properties.getProperty(name);
+                    if (value == null) continue;
+                    // Tests don't need to be changed
+                    if (value.contains("%test_thread_info")) continue;
+                    /*
+                     * Patterns without a marker are sufficiently customized
+                     * that we don't have an opinion about them.
+                     */
+                    if (false == value.contains("%marker")) continue;
+                    if (false == value.contains("%node_name")) {
+                        locationsWithDeprecatedPatterns.add(source.getLocation());
+                        properties.setProperty(name, value.replace("%marker", "[%node_name]%marker "));
+                    }
+                }
+                // end hack
+                return new PropertiesConfigurationBuilder()
+                        .setConfigurationSource(source)
+                        .setRootProperties(properties)
+                        .setLoggerContext(loggerContext)
+                        .build();
+            }
+        };
         final Set<FileVisitOption> options = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
         Files.walkFileTree(configsPath, options, Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
             @Override
@@ -163,6 +226,14 @@ public class LogConfigurator {
         context.start(new CompositeConfiguration(configurations));
 
         configureLoggerLevels(settings);
+
+        final String deprecatedLocationsString = String.join("\n  ", locationsWithDeprecatedPatterns);
+        if (deprecatedLocationsString.length() > 0) {
+            LogManager.getLogger(LogConfigurator.class).warn("Some logging configurations have %marker but don't have %node_name. "
+                    + "We will automatically add %node_name to the pattern to ease the migration for users who customize "
+                    + "log4j2.properties but will stop this behavior in 7.0. You should manually replace `%node_name` with "
+                    + "`[%node_name]%marker ` in these locations:\n  {}", deprecatedLocationsString);
+        }
     }
 
     private static void configureStatusLogger() {
diff --git a/server/src/main/java/org/elasticsearch/common/logging/Loggers.java b/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
index 40983c517c7..58ffe277531 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
@@ -31,13 +31,9 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.node.Node;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
-import static java.util.Arrays.asList;
 import static org.elasticsearch.common.util.CollectionUtils.asArrayList;
 
 /**
@@ -71,29 +67,19 @@ public class Loggers {
     }
 
     public static Logger getLogger(Class<?> clazz, Settings settings, String... prefixes) {
-        final List<String> prefixesList = prefixesList(settings, prefixes);
-        return Loggers.getLogger(clazz, prefixesList.toArray(new String[prefixesList.size()]));
+        return Loggers.getLogger(clazz, prefixes);
     }
 
     public static Logger getLogger(String loggerName, Settings settings, String... prefixes) {
-        final List<String> prefixesList = prefixesList(settings, prefixes);
-        return Loggers.getLogger(loggerName, prefixesList.toArray(new String[prefixesList.size()]));
-    }
-
-    private static List<String> prefixesList(Settings settings, String... prefixes) {
-        List<String> prefixesList = new ArrayList<>();
-        if (Node.NODE_NAME_SETTING.exists(settings)) {
-            prefixesList.add(Node.NODE_NAME_SETTING.get(settings));
-        }
-        if (prefixes != null && prefixes.length > 0) {
-            prefixesList.addAll(asList(prefixes));
-        }
-        return prefixesList;
+        return Loggers.getLogger(loggerName, prefixes);
     }
 
     public static Logger getLogger(Logger parentLogger, String s) {
-        assert parentLogger instanceof PrefixLogger;
-        return ESLoggerFactory.getLogger(((PrefixLogger)parentLogger).prefix(), parentLogger.getName() + s);
+        String prefix = null;
+        if (parentLogger instanceof PrefixLogger) {
+            prefix = ((PrefixLogger)parentLogger).prefix();
+        }
+        return ESLoggerFactory.getLogger(prefix, parentLogger.getName() + s);
     }
 
     public static Logger getLogger(String s) {
@@ -126,7 +112,6 @@ public class Loggers {
                 }
             }
             if (sb.length() > 0) {
-                sb.append(" ");
                 prefix = sb.toString();
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java b/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java
new file mode 100644
index 00000000000..ca4c9ab776f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import java.util.Arrays;
+
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
+import org.apache.logging.log4j.core.pattern.PatternConverter;
+import org.apache.lucene.util.SetOnce;
+
+/**
+ * Converts {@code %node_name} in log4j patterns into the current node name.
+ * We *could* use a system property lookup instead but this is very explicit
+ * and fails fast if we try to use the logger without initializing the node
+ * name. As a bonus it ought to be ever so slightly faster because it doesn't
+ * have to look up the system property every time.
+ */
+@Plugin(category = PatternConverter.CATEGORY, name = "NodeNamePatternConverter")
+@ConverterKeys({"node_name"})
+public class NodeNamePatternConverter extends LogEventPatternConverter {
+    private static final SetOnce<String> NODE_NAME = new SetOnce<>();
+
+    /**
+     * Set the name of this node.
+     */
+    public static void setNodeName(String nodeName) {
+        NODE_NAME.set(nodeName);
+    }
+
+    /**
+     * Called by log4j2 to initialize this converter.
+     */
+    public static NodeNamePatternConverter newInstance(final String[] options) {
+        if (options.length > 0) {
+            throw new IllegalArgumentException("no options supported but options provided: "
+                    + Arrays.toString(options));
+        }
+        return new NodeNamePatternConverter(NODE_NAME.get());
+    }
+
+    private final String nodeName;
+
+    private NodeNamePatternConverter(String nodeName) {
+        super("NodeName", "node_name");
+        this.nodeName = nodeName;
+    }
+
+    @Override
+    public void format(LogEvent event, StringBuilder toAppendTo) {
+        toAppendTo.append(nodeName);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index c1ce864223b..b5cc7964085 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -231,7 +231,14 @@ public class Node implements Closeable {
     }
 
     private static final String CLIENT_TYPE = "node";
+
     private final Lifecycle lifecycle = new Lifecycle();
+
+    /**
+     * Logger initialized in the ctor because if it were initialized statically
+     * then it wouldn't get the node name.
+     */
+    private final Logger logger;
     private final Injector injector;
     private final Settings settings;
     private final Settings originalSettings;
@@ -257,13 +264,9 @@ public class Node implements Closeable {
     }
 
     protected Node(final Environment environment, Collection<Class<? extends Plugin>> classpathPlugins) {
+        logger = Loggers.getLogger(Node.class);
         final List<Closeable> resourcesToClose = new ArrayList<>(); // register everything we need to release in the case of an error
         boolean success = false;
-        {
-            // use temp logger just to say we are starting. we can't use it later on because the node name might not be set
-            Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(environment.settings()));
-            logger.info("initializing ...");
-        }
         try {
             originalSettings = environment.settings();
             Settings tmpSettings = Settings.builder().put(environment.settings())
@@ -279,7 +282,6 @@ public class Node implements Closeable {
             final boolean hadPredefinedNodeName = NODE_NAME_SETTING.exists(tmpSettings);
             final String nodeId = nodeEnvironment.nodeId();
             tmpSettings = addNodeNameIfNeeded(tmpSettings, nodeId);
-            final Logger logger = Loggers.getLogger(Node.class, tmpSettings);
             // this must be captured after the node name is possibly added to the settings
             final String nodeName = NODE_NAME_SETTING.get(tmpSettings);
             if (hadPredefinedNodeName == false) {
@@ -631,7 +633,6 @@ public class Node implements Closeable {
             return this;
         }
 
-        Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("starting ...");
         pluginLifecycleComponents.forEach(LifecycleComponent::start);
 
@@ -742,7 +743,6 @@ public class Node implements Closeable {
         if (!lifecycle.moveToStopped()) {
             return this;
         }
-        Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("stopping ...");
 
         injector.getInstance(ResourceWatcherService.class).stop();
@@ -785,7 +785,6 @@ public class Node implements Closeable {
             return;
         }
 
-        Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("closing ...");
         List<Closeable> toClose = new ArrayList<>();
         StopWatch stopWatch = new StopWatch("node_close");
diff --git a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
index e3ba6d19b39..8180dd96e8e 100644
--- a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
+++ b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
@@ -131,7 +131,7 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e =
                 expectThrows(NodeValidationException.class,
-                    () -> BootstrapChecks.check(defaultContext, true, checks, "testExceptionAggregation"));
+                    () -> BootstrapChecks.check(defaultContext, true, checks));
         assertThat(e, hasToString(allOf(containsString("bootstrap checks failed"), containsString("first"), containsString("second"))));
         final Throwable[] suppressed = e.getSuppressed();
         assertThat(suppressed.length, equalTo(2));
@@ -162,7 +162,7 @@ public class BootstrapChecksTests extends ESTestCase {
         final NodeValidationException e =
                 expectThrows(
                         NodeValidationException.class,
-                        () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testHeapSizeCheck"));
+                        () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(
                 e.getMessage(),
                 containsString("initial heap size [" + initialHeapSize.get() + "] " +
@@ -170,7 +170,7 @@ public class BootstrapChecksTests extends ESTestCase {
 
         initialHeapSize.set(maxHeapSize.get());
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testHeapSizeCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if the initial heap size or the max
         // heap size is not available
@@ -179,7 +179,7 @@ public class BootstrapChecksTests extends ESTestCase {
         } else {
             maxHeapSize.set(0);
         }
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testHeapSizeCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testFileDescriptorLimits() throws NodeValidationException {
@@ -205,17 +205,17 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e =
                 expectThrows(NodeValidationException.class,
-                        () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testFileDescriptorLimits"));
+                        () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max file descriptors"));
 
         maxFileDescriptorCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testFileDescriptorLimits");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if current file descriptor count is
         // not available
         maxFileDescriptorCount.set(-1);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testFileDescriptorLimits");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testFileDescriptorLimitsThrowsOnInvalidLimit() {
@@ -262,15 +262,13 @@ public class BootstrapChecksTests extends ESTestCase {
                         () -> BootstrapChecks.check(
                                 bootstrapContext,
                                 true,
-                                Collections.singletonList(check),
-                                "testFileDescriptorLimitsThrowsOnInvalidLimit"));
+                                Collections.singletonList(check)));
                 assertThat(
                         e.getMessage(),
                         containsString("memory locking requested for elasticsearch process but memory is not locked"));
             } else {
                 // nothing should happen
-                BootstrapChecks.check(bootstrapContext, true, Collections.singletonList(check),
-                    "testFileDescriptorLimitsThrowsOnInvalidLimit");
+                BootstrapChecks.check(bootstrapContext, true, Collections.singletonList(check));
             }
         }
     }
@@ -287,17 +285,17 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max number of threads"));
 
         maxNumberOfThreads.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if current max number of threads is
         // not available
         maxNumberOfThreads.set(-1);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testMaxSizeVirtualMemory() throws NodeValidationException {
@@ -317,16 +315,16 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxSizeVirtualMemory"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max size virtual memory"));
 
         maxSizeVirtualMemory.set(rlimInfinity);
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if max size virtual memory is not available
         maxSizeVirtualMemory.set(Long.MIN_VALUE);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testMaxFileSizeCheck() throws NodeValidationException {
@@ -346,16 +344,16 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxFileSize"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max file size"));
 
         maxFileSize.set(rlimInfinity);
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxFileSize");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if max file size is not available
         maxFileSize.set(Long.MIN_VALUE);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxFileSize");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testMaxMapCountCheck() throws NodeValidationException {
@@ -370,17 +368,17 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxMapCountCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(e.getMessage(), containsString("max virtual memory areas vm.max_map_count"));
 
         maxMapCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
 
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxMapCountCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // nothing should happen if current vm.max_map_count is not
         // available
         maxMapCount.set(-1);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testMaxMapCountCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testClientJvmCheck() throws NodeValidationException {
@@ -394,14 +392,14 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testClientJvmCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(
                 e.getMessage(),
                 containsString("JVM is using the client VM [Java HotSpot(TM) 32-Bit Client VM] " +
                         "but should be using a server VM for the best performance"));
 
         vmName.set("Java HotSpot(TM) 32-Bit Server VM");
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testClientJvmCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testUseSerialGCCheck() throws NodeValidationException {
@@ -415,14 +413,14 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e = expectThrows(
             NodeValidationException.class,
-            () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testUseSerialGCCheck"));
+            () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
         assertThat(
             e.getMessage(),
             containsString("JVM is using the serial collector but should not be for the best performance; " + "" +
                 "either it's the default for the VM [" + JvmInfo.jvmInfo().getVmName() +"] or -XX:+UseSerialGC was explicitly specified"));
 
         useSerialGC.set("false");
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), "testUseSerialGCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
     }
 
     public void testSystemCallFilterCheck() throws NodeValidationException {
@@ -439,15 +437,14 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e = expectThrows(
             NodeValidationException.class,
-            () -> BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck),
-                "testSystemCallFilterCheck"));
+            () -> BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck)));
         assertThat(
             e.getMessage(),
             containsString("system call filters failed to install; " +
                 "check the logs and fix your configuration or disable system call filters at your own risk"));
 
         isSystemCallFilterInstalled.set(true);
-        BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck), "testSystemCallFilterCheck");
+        BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck));
         BootstrapContext context_1 = new BootstrapContext(Settings.builder().put("bootstrap.system_call_filter", false).build(), null);
         final BootstrapChecks.SystemCallFilterCheck systemCallFilterNotEnabledCheck = new BootstrapChecks.SystemCallFilterCheck() {
             @Override
@@ -456,9 +453,9 @@ public class BootstrapChecksTests extends ESTestCase {
             }
         };
         isSystemCallFilterInstalled.set(false);
-        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck), "testSystemCallFilterCheck");
+        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck));
         isSystemCallFilterInstalled.set(true);
-        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck), "testSystemCallFilterCheck");
+        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck));
     }
 
     public void testMightForkCheck() throws NodeValidationException {
@@ -553,8 +550,6 @@ public class BootstrapChecksTests extends ESTestCase {
         final Runnable enableMightFork,
         final Consumer<NodeValidationException> consumer) throws NodeValidationException {
 
-        final String methodName = Thread.currentThread().getStackTrace()[2].getMethodName();
-
         // if system call filter is disabled, nothing should happen
         isSystemCallFilterInstalled.set(false);
         if (randomBoolean()) {
@@ -562,13 +557,13 @@ public class BootstrapChecksTests extends ESTestCase {
         } else {
             enableMightFork.run();
         }
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), methodName);
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // if system call filter is enabled, but we will not fork, nothing should
         // happen
         isSystemCallFilterInstalled.set(true);
         disableMightFork.run();
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check), methodName);
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
 
         // if system call filter is enabled, and we might fork, the check should be enforced, regardless of bootstrap checks being enabled
         // or not
@@ -577,7 +572,7 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException e = expectThrows(
             NodeValidationException.class,
-            () -> BootstrapChecks.check(defaultContext, randomBoolean(), Collections.singletonList(check), methodName));
+            () -> BootstrapChecks.check(defaultContext, randomBoolean(), Collections.singletonList(check)));
         consumer.accept(e);
     }
 
@@ -602,7 +597,7 @@ public class BootstrapChecksTests extends ESTestCase {
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
                 () -> {
-                    BootstrapChecks.check(defaultContext, true, checks, "testEarlyAccessCheck");
+                    BootstrapChecks.check(defaultContext, true, checks);
                 });
         assertThat(
                 e.getMessage(),
@@ -613,7 +608,7 @@ public class BootstrapChecksTests extends ESTestCase {
 
         // if not on an early-access build, nothing should happen
         javaVersion.set(randomFrom("1.8.0_152", "9"));
-        BootstrapChecks.check(defaultContext, true, checks, "testEarlyAccessCheck");
+        BootstrapChecks.check(defaultContext, true, checks);
 
     }
 
@@ -649,7 +644,7 @@ public class BootstrapChecksTests extends ESTestCase {
         final NodeValidationException e =
             expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck), "testG1GCCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck)));
         assertThat(
             e.getMessage(),
             containsString(
@@ -657,12 +652,12 @@ public class BootstrapChecksTests extends ESTestCase {
 
         // if G1GC is disabled, nothing should happen
         isG1GCEnabled.set(false);
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck), "testG1GCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck));
 
         // if on or after update 40, nothing should happen independent of whether or not G1GC is enabled
         isG1GCEnabled.set(randomBoolean());
         jvmVersion.set(String.format(Locale.ROOT, "25.%d-b%d", randomIntBetween(40, 112), randomIntBetween(1, 128)));
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck), "testG1GCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(g1GCCheck));
 
         final BootstrapChecks.G1GCCheck nonOracleCheck = new BootstrapChecks.G1GCCheck() {
 
@@ -674,7 +669,7 @@ public class BootstrapChecksTests extends ESTestCase {
         };
 
         // if not on an Oracle JVM, nothing should happen
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonOracleCheck), "testG1GCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonOracleCheck));
 
         final BootstrapChecks.G1GCCheck nonJava8Check = new BootstrapChecks.G1GCCheck() {
 
@@ -686,7 +681,7 @@ public class BootstrapChecksTests extends ESTestCase {
         };
 
         // if not Java 8, nothing should happen
-        BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonJava8Check), "testG1GCCheck");
+        BootstrapChecks.check(defaultContext, true, Collections.singletonList(nonJava8Check));
     }
 
     public void testAllPermissionCheck() throws NodeValidationException {
@@ -701,12 +696,12 @@ public class BootstrapChecksTests extends ESTestCase {
         final List<BootstrapCheck> checks = Collections.singletonList(allPermissionCheck);
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
-                () -> BootstrapChecks.check(defaultContext, true, checks, "testIsAllPermissionCheck"));
+                () -> BootstrapChecks.check(defaultContext, true, checks));
         assertThat(e, hasToString(containsString("granting the all permission effectively disables security")));
 
         // if all permissions are not granted, nothing should happen
         isAllPermissionGranted.set(false);
-        BootstrapChecks.check(defaultContext, true, checks, "testIsAllPermissionCheck");
+        BootstrapChecks.check(defaultContext, true, checks);
     }
 
     public void testAlwaysEnforcedChecks() {
@@ -724,7 +719,7 @@ public class BootstrapChecksTests extends ESTestCase {
 
         final NodeValidationException alwaysEnforced = expectThrows(
             NodeValidationException.class,
-            () -> BootstrapChecks.check(defaultContext, randomBoolean(), Collections.singletonList(check), "testAlwaysEnforcedChecks"));
+            () -> BootstrapChecks.check(defaultContext, randomBoolean(), Collections.singletonList(check)));
         assertThat(alwaysEnforced, hasToString(containsString("error")));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java b/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
index e2bf07b7d0b..a6e50170d7e 100644
--- a/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
@@ -65,7 +65,7 @@ public class ElasticsearchUncaughtExceptionHandlerTests extends ESTestCase {
         final AtomicInteger observedStatus = new AtomicInteger();
         final AtomicReference<String> threadNameReference = new AtomicReference<>();
         final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
-        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler(() -> "testUncaughtError") {
+        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler() {
 
             @Override
             void halt(int status) {
@@ -106,7 +106,7 @@ public class ElasticsearchUncaughtExceptionHandlerTests extends ESTestCase {
         thread.setName(name);
         final AtomicReference<String> threadNameReference = new AtomicReference<>();
         final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
-        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler(() -> "testUncaughtException") {
+        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler() {
             @Override
             void halt(int status) {
                 fail();
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index f47cd11e1b4..6169f6d12ee 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -1978,7 +1978,7 @@ public class InternalEngineTests extends EngineTestCase {
         @Override
         public void append(LogEvent event) {
             final String formattedMessage = event.getMessage().getFormattedMessage();
-            if (event.getLevel() == Level.TRACE && event.getMarker().getName().contains("[index][0] ")) {
+            if (event.getLevel() == Level.TRACE && event.getMarker().getName().contains("[index][0]")) {
                 if (event.getLoggerName().endsWith(".IW") &&
                     formattedMessage.contains("IW: now apply all deletes")) {
                     sawIndexWriterMessage = true;
diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java b/test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java
new file mode 100644
index 00000000000..35ad331f532
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import java.util.Arrays;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
+import org.apache.logging.log4j.core.pattern.PatternConverter;
+import org.elasticsearch.test.ESIntegTestCase;
+
+/**
+ * Converts {@code %test_thread_info} in log4j patterns into information
+ * based on the loggin thread's name. If that thread is part of an
+ * {@link ESIntegTestCase} then this information is the node name.
+ */
+@Plugin(category = PatternConverter.CATEGORY, name = "TestInfoPatternConverter")
+@ConverterKeys({"test_thread_info"})
+public class TestThreadInfoPatternConverter extends LogEventPatternConverter {
+    /**
+     * Called by log4j2 to initialize this converter.
+     */
+    public static TestThreadInfoPatternConverter newInstance(final String[] options) {
+        if (options.length > 0) {
+            throw new IllegalArgumentException("no options supported but options provided: "
+                    + Arrays.toString(options));
+        }
+        return new TestThreadInfoPatternConverter();
+    }
+
+    private TestThreadInfoPatternConverter() {
+        super("TestInfo", "test_thread_info");
+    }
+
+    @Override
+    public void format(LogEvent event, StringBuilder toAppendTo) {
+        toAppendTo.append(threadInfo(event.getThreadName()));
+    }
+
+    private static final Pattern ELASTICSEARCH_THREAD_NAME_PATTERN =
+            Pattern.compile("elasticsearch\\[(.+)\\]\\[.+\\].+");
+    private static final Pattern TEST_THREAD_NAME_PATTERN =
+            Pattern.compile("TEST-.+\\.(.+)-seed#\\[.+\\]");
+    private static final Pattern TEST_SUITE_INIT_THREAD_NAME_PATTERN =
+            Pattern.compile("SUITE-.+-worker");
+    private static final Pattern NOT_YET_NAMED_NODE_THREAD_NAME_PATTERN =
+            Pattern.compile("test_SUITE-CHILD_VM.+cluster\\[T#(.+)\\]");
+    static String threadInfo(String threadName) {
+        Matcher m = ELASTICSEARCH_THREAD_NAME_PATTERN.matcher(threadName);
+        if (m.matches()) {
+            // Thread looks like a node thread so use the node name
+            return m.group(1);
+        }
+        m = TEST_THREAD_NAME_PATTERN.matcher(threadName);
+        if (m.matches()) {
+            /*
+             * Thread looks like a test thread so use the test method name.
+             * It'd be pretty reasonable not to use a prefix at all here but
+             * the logger layout pretty much expects one and the test method
+             * can be pretty nice to have around anyway.
+             */
+            return m.group(1);
+        }
+        m = TEST_SUITE_INIT_THREAD_NAME_PATTERN.matcher(threadName);
+        if (m.matches()) {
+            /*
+             * Thread looks like test suite initialization or tead down and
+             * we don't have any more information to give. Like above, we
+             * could spit out nothing here but the logger layout expect
+             * something and it *is* nice to know what lines come from test
+             * teardown and initialization.
+             */
+            return "suite";
+        }
+        m = NOT_YET_NAMED_NODE_THREAD_NAME_PATTERN.matcher(threadName);
+        if (m.matches()) {
+            /*
+             * These are as yet unnamed integ test nodes. I'd prefer to log
+             * the node name but I don't have it yet.
+             */
+            return "integ_" + m.group(1) + "";
+        }
+        /*
+         * These are uncategorized threads. We log the entire thread name in
+         * case it is useful. We wrap it in `[]` so you tell that it is a
+         * thread name rather than a node name or something.
+         */
+        return "[" + threadName + "]";
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index ee4f2f3b222..1498a69cf6a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -65,6 +65,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -200,6 +201,8 @@ public abstract class ESTestCase extends LuceneTestCase {
         System.setProperty("log4j.shutdownHookEnabled", "false");
         System.setProperty("log4j2.disable.jmx", "true");
 
+        LogConfigurator.loadLog4jPlugins();
+
         // Enable Netty leak detection and monitor logger for logged leak errors
         System.setProperty("io.netty.leakDetection.level", "paranoid");
         String leakLoggerName = "io.netty.util.ResourceLeakDetector";
@@ -321,7 +324,7 @@ public abstract class ESTestCase extends LuceneTestCase {
 
     @Before
     public final void before()  {
-        logger.info("[{}]: before test", getTestName());
+        logger.info("{}before test", getTestParamsForLogging());
         assertNull("Thread context initialized twice", threadContext);
         if (enableWarningsCheck()) {
             this.threadContext = new ThreadContext(Settings.EMPTY);
@@ -350,7 +353,16 @@ public abstract class ESTestCase extends LuceneTestCase {
         }
         ensureAllSearchContextsReleased();
         ensureCheckIndexPassed();
-        logger.info("[{}]: after test", getTestName());
+        logger.info("{}after test", getTestParamsForLogging());
+    }
+
+    private String getTestParamsForLogging() {
+        String name = getTestName();
+        int start = name.indexOf('{');
+        if (start < 0) return "";
+        int end = name.lastIndexOf('}');
+        if (end < 0) return "";
+        return "[" + name.substring(start + 1, end) + "] ";
     }
 
     private void ensureNoWarnings() throws IOException {
diff --git a/test/framework/src/main/resources/log4j2-test.properties b/test/framework/src/main/resources/log4j2-test.properties
index 828555ed52e..842c9c79d79 100644
--- a/test/framework/src/main/resources/log4j2-test.properties
+++ b/test/framework/src/main/resources/log4j2-test.properties
@@ -1,7 +1,7 @@
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%test_thread_info]%marker %m%n
 
 rootLogger.level = ${sys:tests.es.logger.level:-info}
 rootLogger.appenderRef.console.ref = console
diff --git a/test/framework/src/main/resources/log4j2.component.properties b/test/framework/src/main/resources/log4j2.component.properties
new file mode 100644
index 00000000000..8ce0fb493b7
--- /dev/null
+++ b/test/framework/src/main/resources/log4j2.component.properties
@@ -0,0 +1,9 @@
+# RandomizedRunner renames threads and we'd like to look at accurate thread
+# names. Also, this is the default in log4j2's master branch for versions of
+# Java after 1.8_102 which most of the versions we use are anyway. After that
+# version of Java calling `Thread.getName()` doesn't allocate so UNCACHED
+# ought to be faster than CACHED because it doesn't have to deal with
+# ThreadLocals.
+# While we don't use AsyncLogger, we do end up with mutable log events and
+# those use this key for configuration.
+AsyncLogger.ThreadNameStrategy = UNCACHED
diff --git a/test/framework/src/test/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverterTests.java b/test/framework/src/test/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverterTests.java
new file mode 100644
index 00000000000..8a98b867a46
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverterTests.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
+
+import static org.elasticsearch.common.logging.TestThreadInfoPatternConverter.threadInfo;
+
+public class TestThreadInfoPatternConverterTests extends ESTestCase {
+    private static String suiteInfo;
+
+    @BeforeClass
+    public static void captureSuiteInfo() {
+        suiteInfo = threadInfo(Thread.currentThread().getName());
+    }
+
+    public void testThreadInfo() {
+        // Threads that are part of a node get the node name
+        String nodeName = randomAlphaOfLength(5);
+        String threadName = EsExecutors.threadName(nodeName, randomAlphaOfLength(20))
+                + "[T#" + between(0, 1000) + "]";
+        assertEquals(nodeName, threadInfo(threadName));
+
+        // Test threads get the test name
+        assertEquals(getTestName(), threadInfo(Thread.currentThread().getName()));
+
+        // Suite initalization gets "suite"
+        assertEquals("suite", suiteInfo);
+
+        // And stuff that doesn't match anything gets wrapped in [] so we can see it
+        String unmatched = randomAlphaOfLength(5);
+        assertEquals("[" + unmatched + "]", threadInfo(unmatched));
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java
index 4226607f192..f28fe1c297f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateTool.java
@@ -363,9 +363,16 @@ public class ESNativeRealmMigrateTool extends LoggingAwareMultiCommand {
         final Logger logger = ESLoggerFactory.getLogger(ESNativeRealmMigrateTool.class);
         Loggers.setLevel(logger, Level.ALL);
 
+        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+        final Configuration config = ctx.getConfiguration();
+
         // create appender
         final Appender appender = new AbstractAppender(ESNativeRealmMigrateTool.class.getName(), null,
-                PatternLayout.newBuilder().withPattern("%m").build()) {
+                PatternLayout.newBuilder()
+                    // Specify the configuration so log4j doesn't re-initialize
+                    .withConfiguration(config)
+                    .withPattern("%m")
+                    .build()) {
             @Override
             public void append(LogEvent event) {
                 switch (event.getLevel().getStandardLevel()) {
@@ -384,8 +391,6 @@ public class ESNativeRealmMigrateTool extends LoggingAwareMultiCommand {
         appender.start();
 
         // get the config, detach from parent, remove appenders, add custom appender
-        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
-        final Configuration config = ctx.getConfiguration();
         final LoggerConfig loggerConfig = config.getLoggerConfig(ESNativeRealmMigrateTool.class.getName());
         loggerConfig.setParent(null);
         loggerConfig.getAppenders().forEach((s, a) -> Loggers.removeAppender(logger, a));

From 2ed9782a678003c34ca536207463e2d8f860a74c Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 31 Jul 2018 08:28:03 -0700
Subject: [PATCH 246/260] Scripting: Fix painless compiler loader to know about
 context classes (#32385)

This commit fixes the painless compiler classloader to know about the
classes from the script context. This fixes an issue when a custom
context is used from a plugin which caused a ClassNotFoundException for
the script class and its factory classes.
---
 .../org/elasticsearch/painless/Compiler.java  | 46 +++++++++++++------
 .../painless/LambdaBootstrap.java             |  7 ++-
 .../painless/PainlessScriptEngine.java        |  4 +-
 .../painless/BaseClassTests.java              | 44 +++++++++---------
 .../org/elasticsearch/painless/Debugger.java  |  2 +-
 5 files changed, 61 insertions(+), 42 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 03345fcfff3..807b4409d7a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -69,17 +69,14 @@ final class Compiler {
     /**
      * A secure class loader used to define Painless scripts.
      */
-    static final class Loader extends SecureClassLoader {
+    final class Loader extends SecureClassLoader {
         private final AtomicInteger lambdaCounter = new AtomicInteger(0);
-        private final PainlessLookup painlessLookup;
 
         /**
          * @param parent The parent ClassLoader.
          */
-        Loader(ClassLoader parent, PainlessLookup painlessLookup) {
+        Loader(ClassLoader parent) {
             super(parent);
-
-            this.painlessLookup = painlessLookup;
         }
 
         /**
@@ -90,6 +87,15 @@ final class Compiler {
          */
         @Override
         public Class<?> findClass(String name) throws ClassNotFoundException {
+            if (scriptClass.getName().equals(name)) {
+                return scriptClass;
+            }
+            if (factoryClass != null && factoryClass.getName().equals(name)) {
+                return factoryClass;
+            }
+            if (statefulFactoryClass != null && statefulFactoryClass.getName().equals(name)) {
+                return statefulFactoryClass;
+            }
             Class<?> found = painlessLookup.getClassFromBinaryName(name);
 
             return found != null ? found : super.findClass(name);
@@ -139,13 +145,23 @@ final class Compiler {
      * {@link Compiler}'s specified {@link PainlessLookup}.
      */
     public Loader createLoader(ClassLoader parent) {
-        return new Loader(parent, painlessLookup);
+        return new Loader(parent);
     }
 
     /**
-     * The class/interface the script is guaranteed to derive/implement.
+     * The class/interface the script will implement.
      */
-    private final Class<?> base;
+    private final Class<?> scriptClass;
+
+    /**
+     * The class/interface to create the {@code scriptClass} instance.
+     */
+    private final Class<?> factoryClass;
+
+    /**
+     * An optional class/interface to create the {@code factoryClass} instance.
+     */
+    private final Class<?> statefulFactoryClass;
 
     /**
      * The whitelist the script will use.
@@ -154,11 +170,15 @@ final class Compiler {
 
     /**
      * Standard constructor.
-     * @param base The class/interface the script is guaranteed to derive/implement.
+     * @param scriptClass The class/interface the script will implement.
+     * @param factoryClass An optional class/interface to create the {@code scriptClass} instance.
+     * @param statefulFactoryClass An optional class/interface to create the {@code factoryClass} instance.
      * @param painlessLookup The whitelist the script will use.
      */
-    Compiler(Class<?> base, PainlessLookup painlessLookup) {
-        this.base = base;
+    Compiler(Class<?> scriptClass, Class<?> factoryClass, Class<?> statefulFactoryClass, PainlessLookup painlessLookup) {
+        this.scriptClass = scriptClass;
+        this.factoryClass = factoryClass;
+        this.statefulFactoryClass = statefulFactoryClass;
         this.painlessLookup = painlessLookup;
     }
 
@@ -177,7 +197,7 @@ final class Compiler {
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, base);
+        ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SSource root = Walker.buildPainlessTree(scriptClassInfo, reserved, name, source, settings, painlessLookup,
                 null);
         root.analyze(painlessLookup);
@@ -209,7 +229,7 @@ final class Compiler {
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, base);
+        ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SSource root = Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), name, source, settings, painlessLookup,
                 debugStream);
         root.analyze(painlessLookup);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
index 3fc8554b271..d95dc426688 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
@@ -36,7 +36,6 @@ import java.security.AccessController;
 import java.security.PrivilegedAction;
 
 import static java.lang.invoke.MethodHandles.Lookup;
-import static org.elasticsearch.painless.Compiler.Loader;
 import static org.elasticsearch.painless.WriterConstants.CLASS_VERSION;
 import static org.elasticsearch.painless.WriterConstants.CTOR_METHOD_NAME;
 import static org.elasticsearch.painless.WriterConstants.DELEGATE_BOOTSTRAP_HANDLE;
@@ -207,7 +206,7 @@ public final class LambdaBootstrap {
             MethodType delegateMethodType,
             int isDelegateInterface)
             throws LambdaConversionException {
-        Loader loader = (Loader)lookup.lookupClass().getClassLoader();
+        Compiler.Loader loader = (Compiler.Loader)lookup.lookupClass().getClassLoader();
         String lambdaClassName = Type.getInternalName(lookup.lookupClass()) + "$$Lambda" + loader.newLambdaIdentifier();
         Type lambdaClassType = Type.getObjectType(lambdaClassName);
         Type delegateClassType = Type.getObjectType(delegateClassName.replace('.', '/'));
@@ -457,11 +456,11 @@ public final class LambdaBootstrap {
     }
 
     /**
-     * Defines the {@link Class} for the lambda class using the same {@link Loader}
+     * Defines the {@link Class} for the lambda class using the same {@link Compiler.Loader}
      * that originally defined the class for the Painless script.
      */
     private static Class<?> createLambdaClass(
-            Loader loader,
+            Compiler.Loader loader,
             ClassWriter cw,
             Type lambdaClassType) {
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
index a5a9823d130..3a2a6d1452d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
@@ -102,10 +102,10 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
         for (Map.Entry<ScriptContext<?>, List<Whitelist>> entry : contexts.entrySet()) {
             ScriptContext<?> context = entry.getKey();
             if (context.instanceClazz.equals(SearchScript.class) || context.instanceClazz.equals(ExecutableScript.class)) {
-                contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class,
+                contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class, null, null,
                         PainlessLookupBuilder.buildFromWhitelists(entry.getValue())));
             } else {
-                contextsToCompilers.put(context, new Compiler(context.instanceClazz,
+                contextsToCompilers.put(context, new Compiler(context.instanceClazz, context.factoryClazz, context.statefulFactoryClazz,
                         PainlessLookupBuilder.buildFromWhitelists(entry.getValue())));
             }
         }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
index c852d5a41de..c68302bde56 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java
@@ -69,7 +69,7 @@ public class BaseClassTests extends ScriptTestCase {
     }
 
     public void testGets() {
-        Compiler compiler = new Compiler(Gets.class, painlessLookup);
+        Compiler compiler = new Compiler(Gets.class, null, null, painlessLookup);
         Map<String, Object> map = new HashMap<>();
         map.put("s", 1);
 
@@ -87,7 +87,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute();
     }
     public void testNoArgs() {
-        Compiler compiler = new Compiler(NoArgs.class, painlessLookup);
+        Compiler compiler = new Compiler(NoArgs.class, null, null, painlessLookup);
         assertEquals(1, ((NoArgs)scriptEngine.compile(compiler, null, "1", emptyMap())).execute());
         assertEquals("foo", ((NoArgs)scriptEngine.compile(compiler, null, "'foo'", emptyMap())).execute());
 
@@ -111,13 +111,13 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(Object arg);
     }
     public void testOneArg() {
-        Compiler compiler = new Compiler(OneArg.class, painlessLookup);
+        Compiler compiler = new Compiler(OneArg.class, null, null, painlessLookup);
         Object rando = randomInt();
         assertEquals(rando, ((OneArg)scriptEngine.compile(compiler, null, "arg", emptyMap())).execute(rando));
         rando = randomAlphaOfLength(5);
         assertEquals(rando, ((OneArg)scriptEngine.compile(compiler, null, "arg", emptyMap())).execute(rando));
 
-        Compiler noargs = new Compiler(NoArgs.class, painlessLookup);
+        Compiler noargs = new Compiler(NoArgs.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, () ->
                 scriptEngine.compile(noargs, null, "doc", emptyMap()));
         assertEquals("Variable [doc] is not defined.", e.getMessage());
@@ -132,7 +132,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(String[] arg);
     }
     public void testArrayArg() {
-        Compiler compiler = new Compiler(ArrayArg.class, painlessLookup);
+        Compiler compiler = new Compiler(ArrayArg.class, null, null, painlessLookup);
         String rando = randomAlphaOfLength(5);
         assertEquals(rando, ((ArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new String[] {rando, "foo"}));
     }
@@ -142,7 +142,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(int[] arg);
     }
     public void testPrimitiveArrayArg() {
-        Compiler compiler = new Compiler(PrimitiveArrayArg.class, painlessLookup);
+        Compiler compiler = new Compiler(PrimitiveArrayArg.class, null, null, painlessLookup);
         int rando = randomInt();
         assertEquals(rando, ((PrimitiveArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new int[] {rando, 10}));
     }
@@ -152,7 +152,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(Object[] arg);
     }
     public void testDefArrayArg() {
-        Compiler compiler = new Compiler(DefArrayArg.class, painlessLookup);
+        Compiler compiler = new Compiler(DefArrayArg.class, null, null, painlessLookup);
         Object rando = randomInt();
         assertEquals(rando, ((DefArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new Object[] {rando, 10}));
         rando = randomAlphaOfLength(5);
@@ -170,7 +170,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract boolean needsD();
     }
     public void testManyArgs() {
-        Compiler compiler = new Compiler(ManyArgs.class, painlessLookup);
+        Compiler compiler = new Compiler(ManyArgs.class, null, null, painlessLookup);
         int rando = randomInt();
         assertEquals(rando, ((ManyArgs)scriptEngine.compile(compiler, null, "a", emptyMap())).execute(rando, 0, 0, 0));
         assertEquals(10, ((ManyArgs)scriptEngine.compile(compiler, null, "a + b + c + d", emptyMap())).execute(1, 2, 3, 4));
@@ -198,7 +198,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(String... arg);
     }
     public void testVararg() {
-        Compiler compiler = new Compiler(VarargTest.class, painlessLookup);
+        Compiler compiler = new Compiler(VarargTest.class, null, null, painlessLookup);
         assertEquals("foo bar baz", ((VarargTest)scriptEngine.compile(compiler, null, "String.join(' ', Arrays.asList(arg))", emptyMap()))
                     .execute("foo", "bar", "baz"));
     }
@@ -214,7 +214,7 @@ public class BaseClassTests extends ScriptTestCase {
         }
     }
     public void testDefaultMethods() {
-        Compiler compiler = new Compiler(DefaultMethods.class, painlessLookup);
+        Compiler compiler = new Compiler(DefaultMethods.class, null, null, painlessLookup);
         int rando = randomInt();
         assertEquals(rando, ((DefaultMethods)scriptEngine.compile(compiler, null, "a", emptyMap())).execute(rando, 0, 0, 0));
         assertEquals(rando, ((DefaultMethods)scriptEngine.compile(compiler, null, "a", emptyMap())).executeWithASingleOne(rando, 0, 0));
@@ -228,7 +228,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract void execute(Map<String, Object> map);
     }
     public void testReturnsVoid() {
-        Compiler compiler = new Compiler(ReturnsVoid.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsVoid.class, null, null, painlessLookup);
         Map<String, Object> map = new HashMap<>();
         ((ReturnsVoid)scriptEngine.compile(compiler, null, "map.a = 'foo'", emptyMap())).execute(map);
         assertEquals(singletonMap("a", "foo"), map);
@@ -247,7 +247,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract boolean execute();
     }
     public void testReturnsPrimitiveBoolean() {
-        Compiler compiler = new Compiler(ReturnsPrimitiveBoolean.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsPrimitiveBoolean.class, null, null, painlessLookup);
 
         assertEquals(true, ((ReturnsPrimitiveBoolean)scriptEngine.compile(compiler, null, "true", emptyMap())).execute());
         assertEquals(false, ((ReturnsPrimitiveBoolean)scriptEngine.compile(compiler, null, "false", emptyMap())).execute());
@@ -289,7 +289,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract int execute();
     }
     public void testReturnsPrimitiveInt() {
-        Compiler compiler = new Compiler(ReturnsPrimitiveInt.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsPrimitiveInt.class, null, null, painlessLookup);
 
         assertEquals(1, ((ReturnsPrimitiveInt)scriptEngine.compile(compiler, null, "1", emptyMap())).execute());
         assertEquals(1, ((ReturnsPrimitiveInt)scriptEngine.compile(compiler, null, "(int) 1L", emptyMap())).execute());
@@ -331,7 +331,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract float execute();
     }
     public void testReturnsPrimitiveFloat() {
-        Compiler compiler = new Compiler(ReturnsPrimitiveFloat.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsPrimitiveFloat.class, null, null, painlessLookup);
 
         assertEquals(1.1f, ((ReturnsPrimitiveFloat)scriptEngine.compile(compiler, null, "1.1f", emptyMap())).execute(), 0);
         assertEquals(1.1f, ((ReturnsPrimitiveFloat)scriptEngine.compile(compiler, null, "(float) 1.1d", emptyMap())).execute(), 0);
@@ -362,7 +362,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract double execute();
     }
     public void testReturnsPrimitiveDouble() {
-        Compiler compiler = new Compiler(ReturnsPrimitiveDouble.class, painlessLookup);
+        Compiler compiler = new Compiler(ReturnsPrimitiveDouble.class, null, null, painlessLookup);
 
         assertEquals(1.0, ((ReturnsPrimitiveDouble)scriptEngine.compile(compiler, null, "1", emptyMap())).execute(), 0);
         assertEquals(1.0, ((ReturnsPrimitiveDouble)scriptEngine.compile(compiler, null, "1L", emptyMap())).execute(), 0);
@@ -396,7 +396,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(String foo);
     }
     public void testNoArgumentsConstant() {
-        Compiler compiler = new Compiler(NoArgumentsConstant.class, painlessLookup);
+        Compiler compiler = new Compiler(NoArgumentsConstant.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertThat(e.getMessage(), startsWith(
@@ -409,7 +409,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(String foo);
     }
     public void testWrongArgumentsConstant() {
-        Compiler compiler = new Compiler(WrongArgumentsConstant.class, painlessLookup);
+        Compiler compiler = new Compiler(WrongArgumentsConstant.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertThat(e.getMessage(), startsWith(
@@ -422,7 +422,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(String foo);
     }
     public void testWrongLengthOfArgumentConstant() {
-        Compiler compiler = new Compiler(WrongLengthOfArgumentConstant.class, painlessLookup);
+        Compiler compiler = new Compiler(WrongLengthOfArgumentConstant.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertThat(e.getMessage(), startsWith("[" + WrongLengthOfArgumentConstant.class.getName() + "#ARGUMENTS] has length [2] but ["
@@ -434,7 +434,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(UnknownArgType foo);
     }
     public void testUnknownArgType() {
-        Compiler compiler = new Compiler(UnknownArgType.class, painlessLookup);
+        Compiler compiler = new Compiler(UnknownArgType.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertEquals("[foo] is of unknown type [" + UnknownArgType.class.getName() + ". Painless interfaces can only accept arguments "
@@ -446,7 +446,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract UnknownReturnType execute(String foo);
     }
     public void testUnknownReturnType() {
-        Compiler compiler = new Compiler(UnknownReturnType.class, painlessLookup);
+        Compiler compiler = new Compiler(UnknownReturnType.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertEquals("Painless can only implement execute methods returning a whitelisted type but [" + UnknownReturnType.class.getName()
@@ -458,7 +458,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(UnknownArgTypeInArray[] foo);
     }
     public void testUnknownArgTypeInArray() {
-        Compiler compiler = new Compiler(UnknownArgTypeInArray.class, painlessLookup);
+        Compiler compiler = new Compiler(UnknownArgTypeInArray.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "1", emptyMap()));
         assertEquals("[foo] is of unknown type [" + UnknownArgTypeInArray.class.getName() + ". Painless interfaces can only accept "
@@ -470,7 +470,7 @@ public class BaseClassTests extends ScriptTestCase {
         public abstract Object execute(boolean foo);
     }
     public void testTwoExecuteMethods() {
-        Compiler compiler = new Compiler(TwoExecuteMethods.class, painlessLookup);
+        Compiler compiler = new Compiler(TwoExecuteMethods.class, null, null, painlessLookup);
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () ->
             scriptEngine.compile(compiler, null, "null", emptyMap()));
         assertEquals("Painless can only implement interfaces that have a single method named [execute] but ["
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index 48af3898e09..ae33ebfb6e9 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -40,7 +40,7 @@ final class Debugger {
         PrintWriter outputWriter = new PrintWriter(output);
         Textifier textifier = new Textifier();
         try {
-            new Compiler(iface, PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS))
+            new Compiler(iface, null, null, PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS))
                     .compile("<debugging>", source, settings, textifier);
         } catch (RuntimeException e) {
             textifier.print(outputWriter);

From b88b3d81c87c27f7ed5e5b4690f38c96574760ce Mon Sep 17 00:00:00 2001
From: Dimitris Athanasiou <dimitris@elastic.co>
Date: Tue, 31 Jul 2018 17:00:55 +0100
Subject: [PATCH 247/260] [ML][DOCS] Fix typo applied_to => applies_to

---
 x-pack/docs/en/rest-api/ml/jobresource.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
index fb3424cab3f..e2d35082fb8 100644
--- a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
+++ b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc
@@ -467,7 +467,7 @@ A condition has the following properties:
     (string) Specifies the condition operator. The available options are
     `gt` (greater than), `gte` (greater than or equals), `lt` (less than) and `lte` (less than or equals).
 `value`:::
-    (double) The value that is compared against the `applied_to` field using the `operator`.
+    (double) The value that is compared against the `applies_to` field using the `operator`.
 --
 
 A rule is required to either have a non-empty scope or at least one condition.

From 67d53e509361aa6795089a8c5fe01c771c006fba Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Tue, 31 Jul 2018 12:45:30 -0400
Subject: [PATCH 248/260] Mute testFilterCacheStats

Tracked at #32506
---
 .../test/java/org/elasticsearch/indices/stats/IndexStatsIT.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 4d0d0d6ff3f..5ed4b370307 100644
--- a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -1004,6 +1004,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         assertEquals(total, shardTotal);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32506")
     public void testFilterCacheStats() throws Exception {
         assertAcked(prepareCreate("index").setSettings(Settings.builder().put(indexSettings()).put("number_of_replicas", 0).build()).get());
         indexRandom(true,

From 5fd72028087a08eb38235adb91cca671b16bcae0 Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Tue, 31 Jul 2018 11:06:00 -0700
Subject: [PATCH 249/260] SQL: Add test for handling of partial results
 (#32474)

Verifies that partial results are rejected by SQL requests.

Closes #32284
---
 .../qa/sql/nosecurity/JdbcShardFailureIT.java | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java

diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java b/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java
new file mode 100644
index 00000000000..e88b6b94d20
--- /dev/null
+++ b/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.qa.sql.nosecurity;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.xpack.qa.sql.jdbc.JdbcIntegrationTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class JdbcShardFailureIT extends JdbcIntegrationTestCase {
+    @Before
+    public void createTestIndex() throws IOException {
+        Request createTest1 = new Request("PUT", "/test1");
+        String body1 = "{\"aliases\":{\"test\":{}}, \"mappings\": {\"doc\": {\"properties\": {\"test_field\":{\"type\":\"integer\"}}}}}";
+        createTest1.setJsonEntity(body1);
+        client().performRequest(createTest1);
+
+        Request createTest2 = new Request("PUT", "/test2");
+        String body2 = "{\"aliases\":{\"test\":{}}, \"mappings\": {\"doc\": {\"properties\": {\"test_field\":{\"type\":\"integer\"}}}}," +
+            "\"settings\": {\"index.routing.allocation.include.node\": \"nowhere\"}}";
+        createTest2.setJsonEntity(body2);
+        createTest2.addParameter("timeout", "100ms");
+        client().performRequest(createTest2);
+
+        Request request = new Request("PUT", "/test1/doc/_bulk");
+        request.addParameter("refresh", "true");
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < 20; i++) {
+            bulk.append("{\"index\":{}}\n");
+            bulk.append("{\"test_field\":" + i + "}\n");
+        }
+        request.setJsonEntity(bulk.toString());
+        client().performRequest(request);
+    }
+
+    public void testPartialResponseHandling() throws SQLException {
+        try (Connection c = esJdbc(); Statement s = c.createStatement()) {
+            SQLException exception = expectThrows(SQLException.class, () -> s.executeQuery("SELECT * FROM test ORDER BY test_field ASC"));
+            assertThat(exception.getMessage(), containsString("Search rejected due to missing shards"));
+        }
+    }
+}

From 0788188574583c6b3cabb27b374b0ea6f97f5ee7 Mon Sep 17 00:00:00 2001
From: Jay Modi <jaymode@users.noreply.github.com>
Date: Tue, 31 Jul 2018 12:16:22 -0600
Subject: [PATCH 250/260] Add licensing enforcement for FIPS mode (#32437)

This commit adds licensing enforcement for FIPS mode through the use of
a bootstrap check, a node join validator, and a check in the license
service. The work done here is based on the current implementation of
the TLS enforcement with a production license.

The bootstrap check is always enforced since we need to enforce the
licensing and this is the best option to do so at the present time.
---
 .../elasticsearch/license/LicenseService.java | 64 +++++++++--------
 .../xpack/core/XPackSettings.java             |  4 ++
 .../license/LicenseFIPSTests.java             | 72 +++++++++++++++++++
 .../FIPS140JKSKeystoreBootstrapCheck.java     |  3 +-
 .../FIPS140LicenseBootstrapCheck.java         | 40 +++++++++++
 ...asswordHashingAlgorithmBootstrapCheck.java |  2 +-
 .../FIPS140SecureSettingsBootstrapCheck.java  |  3 +-
 .../xpack/security/Security.java              | 28 ++++++--
 .../FIPS140LicenseBootstrapCheckTests.java    | 44 ++++++++++++
 ...rdHashingAlgorithmBootstrapCheckTests.java |  6 +-
 .../xpack/security/SecurityTests.java         | 20 ++++++
 11 files changed, 247 insertions(+), 39 deletions(-)
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheck.java
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheckTests.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
index 505d2c55fa6..d299406aad0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
@@ -209,38 +209,44 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
                 }
             }
 
-            if (newLicense.isProductionLicense()
-                    && XPackSettings.SECURITY_ENABLED.get(settings)
-                    && XPackSettings.TRANSPORT_SSL_ENABLED.get(settings) == false
-                    && isProductionMode(settings, clusterService.localNode())) {
-                // security is on but TLS is not configured we gonna fail the entire request and throw an exception
-                throw new IllegalStateException("Cannot install a [" + newLicense.operationMode() +
-                        "] license unless TLS is configured or security is disabled");
+            if (XPackSettings.SECURITY_ENABLED.get(settings)) {
                 // TODO we should really validate that all nodes have xpack installed and are consistently configured but this
                 // should happen on a different level and not in this code
-            } else {
-                clusterService.submitStateUpdateTask("register license [" + newLicense.uid() + "]", new
-                        AckedClusterStateUpdateTask<PutLicenseResponse>(request, listener) {
-                            @Override
-                            protected PutLicenseResponse newResponse(boolean acknowledged) {
-                                return new PutLicenseResponse(acknowledged, LicensesStatus.VALID);
-                            }
-
-                            @Override
-                            public ClusterState execute(ClusterState currentState) throws Exception {
-                                XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
-                                MetaData currentMetadata = currentState.metaData();
-                                LicensesMetaData licensesMetaData = currentMetadata.custom(LicensesMetaData.TYPE);
-                                Version trialVersion = null;
-                                if (licensesMetaData != null) {
-                                    trialVersion = licensesMetaData.getMostRecentTrialVersion();
-                                }
-                                MetaData.Builder mdBuilder = MetaData.builder(currentMetadata);
-                                mdBuilder.putCustom(LicensesMetaData.TYPE, new LicensesMetaData(newLicense, trialVersion));
-                                return ClusterState.builder(currentState).metaData(mdBuilder).build();
-                            }
-                        });
+                if (newLicense.isProductionLicense()
+                    && XPackSettings.TRANSPORT_SSL_ENABLED.get(settings) == false
+                    && isProductionMode(settings, clusterService.localNode())) {
+                    // security is on but TLS is not configured we gonna fail the entire request and throw an exception
+                    throw new IllegalStateException("Cannot install a [" + newLicense.operationMode() +
+                        "] license unless TLS is configured or security is disabled");
+                } else if (XPackSettings.FIPS_MODE_ENABLED.get(settings)
+                    && newLicense.operationMode() != License.OperationMode.PLATINUM
+                    && newLicense.operationMode() != License.OperationMode.TRIAL) {
+                    throw new IllegalStateException("Cannot install a [" + newLicense.operationMode() +
+                        "] license unless FIPS mode is disabled");
+                }
             }
+
+            clusterService.submitStateUpdateTask("register license [" + newLicense.uid() + "]", new
+                    AckedClusterStateUpdateTask<PutLicenseResponse>(request, listener) {
+                        @Override
+                        protected PutLicenseResponse newResponse(boolean acknowledged) {
+                            return new PutLicenseResponse(acknowledged, LicensesStatus.VALID);
+                        }
+
+                        @Override
+                        public ClusterState execute(ClusterState currentState) throws Exception {
+                            XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
+                            MetaData currentMetadata = currentState.metaData();
+                            LicensesMetaData licensesMetaData = currentMetadata.custom(LicensesMetaData.TYPE);
+                            Version trialVersion = null;
+                            if (licensesMetaData != null) {
+                                trialVersion = licensesMetaData.getMostRecentTrialVersion();
+                            }
+                            MetaData.Builder mdBuilder = MetaData.builder(currentMetadata);
+                            mdBuilder.putCustom(LicensesMetaData.TYPE, new LicensesMetaData(newLicense, trialVersion));
+                            return ClusterState.builder(currentState).metaData(mdBuilder).build();
+                        }
+                    });
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index 8c4c5e2c760..1a80c6d873b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -87,6 +87,10 @@ public class XPackSettings {
     public static final Setting<Boolean> TOKEN_SERVICE_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.token.enabled",
         XPackSettings.HTTP_SSL_ENABLED::getRaw, Setting.Property.NodeScope);
 
+    /** Setting for enabling or disabling FIPS mode. Defaults to false */
+    public static final Setting<Boolean> FIPS_MODE_ENABLED =
+        Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope);
+
     /** Setting for enabling or disabling sql. Defaults to true. */
     public static final Setting<Boolean> SQL_ENABLED = Setting.boolSetting("xpack.sql.enabled", true, Setting.Property.NodeScope);
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java
new file mode 100644
index 00000000000..c432a207fcb
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.license;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.verify;
+
+public class LicenseFIPSTests extends AbstractLicenseServiceTestCase {
+
+    public void testFIPSCheckWithAllowedLicense() throws Exception {
+        License newLicense = TestUtils.generateSignedLicense(randomFrom("trial", "platinum"), TimeValue.timeValueHours(24L));
+        PutLicenseRequest request = new PutLicenseRequest();
+        request.acknowledge(true);
+        request.license(newLicense);
+        Settings settings = Settings.builder()
+            .put("xpack.security.enabled", true)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.fips_mode.enabled", randomBoolean())
+            .build();
+        XPackLicenseState licenseState = new XPackLicenseState(settings);
+
+        setInitialState(null, licenseState, settings);
+        licenseService.start();
+        PlainActionFuture<PutLicenseResponse> responseFuture = new PlainActionFuture<>();
+        licenseService.registerLicense(request, responseFuture);
+        verify(clusterService).submitStateUpdateTask(any(String.class), any(ClusterStateUpdateTask.class));
+    }
+
+    public void testFIPSCheckWithoutAllowedLicense() throws Exception {
+        License newLicense = TestUtils.generateSignedLicense(randomFrom("gold", "standard"), TimeValue.timeValueHours(24L));
+        PutLicenseRequest request = new PutLicenseRequest();
+        request.acknowledge(true);
+        request.license(newLicense);
+        Settings settings = Settings.builder()
+            .put("xpack.security.enabled", true)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.fips_mode.enabled", true)
+            .build();
+        XPackLicenseState licenseState = new XPackLicenseState(settings);
+
+        setInitialState(null, licenseState, settings);
+        licenseService.start();
+        PlainActionFuture<PutLicenseResponse> responseFuture = new PlainActionFuture<>();
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> licenseService.registerLicense(request, responseFuture));
+        assertThat(e.getMessage(),
+            containsString("Cannot install a [" + newLicense.operationMode() + "] license unless FIPS mode is disabled"));
+        licenseService.stop();
+
+        settings = Settings.builder()
+            .put("xpack.security.enabled", true)
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.fips_mode.enabled", false)
+            .build();
+        licenseState = new XPackLicenseState(settings);
+
+        setInitialState(null, licenseState, settings);
+        licenseService.start();
+        licenseService.registerLicense(request, responseFuture);
+        verify(clusterService).submitStateUpdateTask(any(String.class), any(ClusterStateUpdateTask.class));
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
index 4a2c7b97195..cd5a720eef1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140JKSKeystoreBootstrapCheck.java
@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.security;
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.bootstrap.BootstrapContext;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.XPackSettings;
 
 
 public class FIPS140JKSKeystoreBootstrapCheck implements BootstrapCheck {
@@ -15,7 +16,7 @@ public class FIPS140JKSKeystoreBootstrapCheck implements BootstrapCheck {
     private final boolean fipsModeEnabled;
 
     FIPS140JKSKeystoreBootstrapCheck(Settings settings) {
-        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+        this.fipsModeEnabled = XPackSettings.FIPS_MODE_ENABLED.get(settings);
     }
 
     /**
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheck.java
new file mode 100644
index 00000000000..d1bce0dcdd2
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheck.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicenseService;
+
+import java.util.EnumSet;
+
+/**
+ * A bootstrap check which enforces the licensing of FIPS
+ */
+final class FIPS140LicenseBootstrapCheck implements BootstrapCheck {
+
+    static final EnumSet<License.OperationMode> ALLOWED_LICENSE_OPERATION_MODES =
+        EnumSet.of(License.OperationMode.PLATINUM, License.OperationMode.TRIAL);
+
+    private final boolean isInFipsMode;
+
+    FIPS140LicenseBootstrapCheck(boolean isInFipsMode) {
+        this.isInFipsMode = isInFipsMode;
+    }
+
+    @Override
+    public BootstrapCheckResult check(BootstrapContext context) {
+        if (isInFipsMode) {
+            License license = LicenseService.getLicense(context.metaData);
+            if (license != null && ALLOWED_LICENSE_OPERATION_MODES.contains(license.operationMode()) == false) {
+                return BootstrapCheckResult.failure("FIPS mode is only allowed with a Platinum or Trial license");
+            }
+        }
+        return BootstrapCheckResult.success();
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
index 57e12a211a3..751d63be4fb 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheck.java
@@ -17,7 +17,7 @@ public class FIPS140PasswordHashingAlgorithmBootstrapCheck implements BootstrapC
     private final boolean fipsModeEnabled;
 
     FIPS140PasswordHashingAlgorithmBootstrapCheck(final Settings settings) {
-        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+        this.fipsModeEnabled = XPackSettings.FIPS_MODE_ENABLED.get(settings);
     }
 
     /**
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
index c766dd0ffaa..a8cb32d545e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/FIPS140SecureSettingsBootstrapCheck.java
@@ -10,6 +10,7 @@ import org.elasticsearch.bootstrap.BootstrapContext;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.XPackSettings;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -20,7 +21,7 @@ public class FIPS140SecureSettingsBootstrapCheck implements BootstrapCheck {
     private final Environment environment;
 
     FIPS140SecureSettingsBootstrapCheck(Settings settings, Environment environment) {
-        this.fipsModeEnabled = Security.FIPS_MODE_ENABLED.get(settings);
+        this.fipsModeEnabled = XPackSettings.FIPS_MODE_ENABLED.get(settings);
         this.environment = environment;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index df5e8dcae1b..18bcfeb94a5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -255,8 +255,6 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         DiscoveryPlugin, MapperPlugin, ExtensiblePlugin {
 
     private static final Logger logger = Loggers.getLogger(Security.class);
-    static final Setting<Boolean> FIPS_MODE_ENABLED =
-        Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope);
 
     static final Setting<List<String>> AUDIT_OUTPUTS_SETTING =
         Setting.listSetting(SecurityField.setting("audit.outputs"),
@@ -593,7 +591,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         }
 
         // The following just apply in node mode
-        settingsList.add(FIPS_MODE_ENABLED);
+        settingsList.add(XPackSettings.FIPS_MODE_ENABLED);
 
         // IP Filter settings
         IPFilter.addSettings(settingsList);
@@ -1000,7 +998,8 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
             return new ValidateTLSOnJoin(XPackSettings.TRANSPORT_SSL_ENABLED.get(settings),
                     DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings))
                 .andThen(new ValidateUpgradedSecurityIndex())
-                .andThen(new ValidateLicenseCanBeDeserialized());
+                .andThen(new ValidateLicenseCanBeDeserialized())
+                .andThen(new ValidateLicenseForFIPS(XPackSettings.FIPS_MODE_ENABLED.get(settings)));
         }
         return null;
     }
@@ -1048,6 +1047,27 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         }
     }
 
+    static final class ValidateLicenseForFIPS implements BiConsumer<DiscoveryNode, ClusterState> {
+        private final boolean inFipsMode;
+
+        ValidateLicenseForFIPS(boolean inFipsMode) {
+            this.inFipsMode = inFipsMode;
+        }
+
+        @Override
+        public void accept(DiscoveryNode node, ClusterState state) {
+            if (inFipsMode) {
+                License license = LicenseService.getLicense(state.metaData());
+                if (license != null &&
+                    FIPS140LicenseBootstrapCheck.ALLOWED_LICENSE_OPERATION_MODES.contains(license.operationMode()) == false) {
+                    throw new IllegalStateException("FIPS mode cannot be used with a [" + license.operationMode() +
+                        "] license. It is only allowed with a Platinum or Trial license.");
+
+                }
+            }
+        }
+    }
+
     @Override
     public void reloadSPI(ClassLoader loader) {
         securityExtensions.addAll(SecurityExtension.loadExtensions(loader));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheckTests.java
new file mode 100644
index 00000000000..a2ec8f9fb20
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140LicenseBootstrapCheckTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.TestUtils;
+import org.elasticsearch.test.ESTestCase;
+
+public class FIPS140LicenseBootstrapCheckTests extends ESTestCase {
+
+    public void testBootstrapCheck() throws Exception {
+        assertTrue(new FIPS140LicenseBootstrapCheck(false)
+            .check(new BootstrapContext(Settings.EMPTY, MetaData.EMPTY_META_DATA)).isSuccess());
+        assertTrue(new FIPS140LicenseBootstrapCheck(randomBoolean())
+            .check(new BootstrapContext(Settings.EMPTY, MetaData.EMPTY_META_DATA)).isSuccess());
+
+        License license = TestUtils.generateSignedLicense(TimeValue.timeValueHours(24));
+        MetaData.Builder builder = MetaData.builder();
+        TestUtils.putLicense(builder, license);
+        MetaData metaData = builder.build();
+        if (FIPS140LicenseBootstrapCheck.ALLOWED_LICENSE_OPERATION_MODES.contains(license.operationMode())) {
+            assertTrue(new FIPS140LicenseBootstrapCheck(true).check(new BootstrapContext(
+                Settings.builder().put("xpack.security.fips_mode.enabled", true).build(), metaData)).isSuccess());
+            assertTrue(new FIPS140LicenseBootstrapCheck(false).check(new BootstrapContext(
+                Settings.builder().put("xpack.security.fips_mode.enabled", false).build(), metaData)).isSuccess());
+        } else {
+            assertTrue(new FIPS140LicenseBootstrapCheck(false).check(new BootstrapContext(
+                Settings.builder().put("xpack.security.fips_mode.enabled", false).build(), metaData)).isSuccess());
+            assertTrue(new FIPS140LicenseBootstrapCheck(true).check(new BootstrapContext(
+                Settings.builder().put("xpack.security.fips_mode.enabled", true).build(), metaData)).isFailure());
+            assertEquals("FIPS mode is only allowed with a Platinum or Trial license",
+                new FIPS140LicenseBootstrapCheck(true).check(new BootstrapContext(
+                    Settings.builder().put("xpack.security.fips_mode.enabled", true).build(), metaData)).getMessage());
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
index b424986876a..8632400866a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/FIPS140PasswordHashingAlgorithmBootstrapCheckTests.java
@@ -21,7 +21,7 @@ public class FIPS140PasswordHashingAlgorithmBootstrapCheckTests extends ESTestCa
     public void testPBKDF2AlgorithmIsAllowed() {
         {
             final Settings settings = Settings.builder()
-                    .put(Security.FIPS_MODE_ENABLED.getKey(), true)
+                    .put(XPackSettings.FIPS_MODE_ENABLED.getKey(), true)
                     .put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2_10000")
                     .build();
             final BootstrapCheck.BootstrapCheckResult result =
@@ -31,7 +31,7 @@ public class FIPS140PasswordHashingAlgorithmBootstrapCheckTests extends ESTestCa
 
         {
             final Settings settings = Settings.builder()
-                    .put(Security.FIPS_MODE_ENABLED.getKey(), true)
+                    .put(XPackSettings.FIPS_MODE_ENABLED.getKey(), true)
                     .put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2")
                     .build();
             final BootstrapCheck.BootstrapCheckResult result =
@@ -49,7 +49,7 @@ public class FIPS140PasswordHashingAlgorithmBootstrapCheckTests extends ESTestCa
     }
 
     private void runBCRYPTTest(final boolean fipsModeEnabled, final String passwordHashingAlgorithm) {
-        final Settings.Builder builder = Settings.builder().put(Security.FIPS_MODE_ENABLED.getKey(), fipsModeEnabled);
+        final Settings.Builder builder = Settings.builder().put(XPackSettings.FIPS_MODE_ENABLED.getKey(), fipsModeEnabled);
         if (passwordHashingAlgorithm != null) {
             builder.put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), passwordHashingAlgorithm);
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index e88b1905a7a..857b1694ac8 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -292,6 +292,26 @@ public class SecurityTests extends ESTestCase {
         assertThat(e.getMessage(), containsString("cannot deserialize the license format"));
     }
 
+    public void testJoinValidatorForFIPSLicense() throws Exception {
+        DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(),
+            VersionUtils.randomVersionBetween(random(), null, Version.CURRENT));
+        MetaData.Builder builder = MetaData.builder();
+        License license = TestUtils.generateSignedLicense(TimeValue.timeValueHours(24));
+        TestUtils.putLicense(builder, license);
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(builder.build()).build();
+        new Security.ValidateLicenseForFIPS(false).accept(node, state);
+
+        final boolean isLicenseValidForFips =
+            FIPS140LicenseBootstrapCheck.ALLOWED_LICENSE_OPERATION_MODES.contains(license.operationMode());
+        if (isLicenseValidForFips) {
+            new Security.ValidateLicenseForFIPS(true).accept(node, state);
+        } else {
+            IllegalStateException e = expectThrows(IllegalStateException.class,
+                () -> new Security.ValidateLicenseForFIPS(true).accept(node, state));
+            assertThat(e.getMessage(), containsString("FIPS mode cannot be used"));
+        }
+    }
+
     public void testIndexJoinValidator_Old_And_Rolling() throws Exception {
         createComponents(Settings.EMPTY);
         BiConsumer<DiscoveryNode, ClusterState> joinValidator = security.getJoinValidator();

From b2c2c94741b991575a4bd3616aba4c3de3b33f10 Mon Sep 17 00:00:00 2001
From: aaronoah <aaron.elite1993@gmail.com>
Date: Mon, 30 Jul 2018 19:42:18 -0500
Subject: [PATCH 251/260] Docs: Add all JDKs to CONTRIBUTING.md

---
 CONTRIBUTING.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8ee03c3d999..c591459f01b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -100,6 +100,12 @@ JDK 10 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 8 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 8.
 
+> Note: It is also required to have `JAVA7_HOME`, `JAVA8_HOME` and
+`JAVA10_HOME` available so that the tests can pass.
+
+> Warning: do not use `sdkman` for Java installations which do not have proper
+`jrunscript` for jdk distributions.
+
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
 using the wrapper via the `gradlew` script in the root of the repository.
 

From 21eb9695afe422cbdbbc7af098c602cb74dbfe50 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 31 Jul 2018 17:31:13 -0400
Subject: [PATCH 252/260] Build: Remove shadowing from benchmarks (#32475)

Removes shadowing from the benchmarks. It isn't *strictly* needed. We do
have to rework the documentation on how to run the benchmark, but it
still seems to work if you run everything through gradle.
---
 benchmarks/README.md          | 37 +++++++++++++++++++----------------
 benchmarks/build.gradle       | 28 +++-----------------------
 client/benchmark/README.md    | 35 +++++++++++++++++++++------------
 client/benchmark/build.gradle |  4 ----
 4 files changed, 45 insertions(+), 59 deletions(-)

diff --git a/benchmarks/README.md b/benchmarks/README.md
index 03aaac7f3c4..5be14bab0ec 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -4,36 +4,39 @@ This directory contains the microbenchmark suite of Elasticsearch. It relies on
 
 ## Purpose
 
-We do not want to microbenchmark everything but the kitchen sink and should typically rely on our 
-[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with 
-[Rally](http://github.com/elastic/rally). Microbenchmarks are intended to spot performance regressions in performance-critical components. 
+We do not want to microbenchmark everything but the kitchen sink and should typically rely on our
+[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with
+[Rally](http://github.com/elastic/rally). Microbenchmarks are intended to spot performance regressions in performance-critical components.
 The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR.
 
 ## Getting Started
 
-Just run `gradle :benchmarks:jmh` from the project root directory. It will build all microbenchmarks, execute them and print the result.
+Just run `gradlew -p benchmarks run` from the project root
+directory. It will build all microbenchmarks, execute them and print
+the result.
 
 ## Running Microbenchmarks
 
-Benchmarks are always run via Gradle with `gradle :benchmarks:jmh`.
- 
-Running via an IDE is not supported as the results are meaningless (we have no control over the JVM running the benchmarks).
+Running via an IDE is not supported as the results are meaningless
+because we have no control over the JVM running the benchmarks.
 
-If you want to run a specific benchmark class, e.g. `org.elasticsearch.benchmark.MySampleBenchmark` or have special requirements 
-generate the uberjar with `gradle :benchmarks:jmhJar` and run it directly with:
+If you want to run a specific benchmark class like, say,
+`MemoryStatsBenchmark`, you can use `--args`:
 
 ```
-java -jar benchmarks/build/distributions/elasticsearch-benchmarks-*.jar
+gradlew -p benchmarks run --args ' MemoryStatsBenchmark'
 ```
 
-JMH supports lots of command line parameters. Add `-h` to the command above to see the available command line options.
+Everything in the `'` gets sent on the command line to JMH. The leading ` `
+inside the `'`s is important. Without it parameters are sometimes sent to
+gradle.
 
 ## Adding Microbenchmarks
 
-Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the 
+Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the
 [JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/).
 
-In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and 
+In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and
 end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`.
 
 ## Tips and Best Practices
@@ -42,15 +45,15 @@ To get realistic results, you should exercise care when running benchmarks. Here
 
 ### Do
 
-* Ensure that the system executing your microbenchmarks has as little load as possible. Shutdown every process that can cause unnecessary 
+* Ensure that the system executing your microbenchmarks has as little load as possible. Shutdown every process that can cause unnecessary
   runtime jitter. Watch the `Error` column in the benchmark results to see the run-to-run variance.
 * Ensure to run enough warmup iterations to get the benchmark into a stable state. If you are unsure, don't change the defaults.
 * Avoid CPU migrations by pinning your benchmarks to specific CPU cores. On Linux you can use `taskset`.
-* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the 
+* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the
   `performance` CPU governor.
 * Vary the problem input size with `@Param`.
 * Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses:
-    * Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews 
+    * Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews
    your results. If so, try to force a GC between runs (`-gc true`) but watch out for the caveats.
     * Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots.
 * Have your benchmarks peer-reviewed.
@@ -59,4 +62,4 @@ To get realistic results, you should exercise care when running benchmarks. Here
 
 * Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.g. with `-prof perfasm`.
 * Run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks).
-* Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable.
\ No newline at end of file
+* Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable.
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index 80d1982300d..0838af72871 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -18,11 +18,8 @@
  */
 
 apply plugin: 'elasticsearch.build'
-
-// order of this section matters, see: https://github.com/johnrengelman/shadow/issues/336
-apply plugin: 'application' // have the shadow plugin provide the runShadow task
+apply plugin: 'application'
 mainClassName = 'org.openjdk.jmh.Main'
-apply plugin: 'com.github.johnrengelman.shadow' // build an uberjar with all benchmarks
 
 // Not published so no need to assemble
 tasks.remove(assemble)
@@ -50,10 +47,8 @@ compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-u
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
 
-forbiddenApis {
-    // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
-    ignoreFailures = true
-}
+// classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
+forbiddenApisMain.enabled = false
 
 // No licenses for our benchmark deps (we don't ship benchmarks)
 dependencyLicenses.enabled = false
@@ -69,20 +64,3 @@ thirdPartyAudit.excludes = [
         'org.openjdk.jmh.profile.HotspotRuntimeProfiler',
         'org.openjdk.jmh.util.Utils'
 ]
-
-runShadow {
-    executable = new File(project.runtimeJavaHome, 'bin/java')
-}
-
-// alias the shadowJar and runShadow tasks to abstract from the concrete plugin that we are using and provide a more consistent interface
-task jmhJar(
-        dependsOn: shadowJar,
-        description: 'Generates an uberjar with the microbenchmarks and all dependencies',
-        group: 'Benchmark'
-)
-
-task jmh(
-        dependsOn: runShadow,
-        description: 'Runs all microbenchmarks',
-        group: 'Benchmark'
-)
diff --git a/client/benchmark/README.md b/client/benchmark/README.md
index 06211b9d8fe..68a910468e0 100644
--- a/client/benchmark/README.md
+++ b/client/benchmark/README.md
@@ -2,10 +2,18 @@
 
 1. Build `client-benchmark-noop-api-plugin` with `gradle :client:client-benchmark-noop-api-plugin:assemble`
 2. Install it on the target host with `bin/elasticsearch-plugin install file:///full/path/to/client-benchmark-noop-api-plugin.zip`
-3. Start Elasticsearch on the target host (ideally *not* on the same machine)
-4. Build an uberjar with `gradle :client:benchmark:shadowJar` and execute it.
+3. Start Elasticsearch on the target host (ideally *not* on the machine
+that runs the benchmarks)
+4. Run the benchmark with
+```
+./gradlew -p client/benchmark run --args ' params go here'
+```
 
-Repeat all steps above for the other benchmark candidate.
+Everything in the `'` gets sent on the command line to JMH. The leading ` `
+inside the `'`s is important. Without it parameters are sometimes sent to
+gradle.
+
+See below for some example invocations.
 
 ### Example benchmark
 
@@ -13,32 +21,35 @@ In general, you should define a few GC-related settings `-Xms8192M -Xmx8192M -XX
 
 #### Bulk indexing
 
-Download benchmark data from http://benchmarks.elastic.co/corpora/geonames/documents.json.bz2 and decompress them.
+Download benchmark data from http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora/geonames and decompress them.
 
-Example command line parameters:
+Example invocation:
 
 ```
-rest bulk 192.168.2.2 ./documents.json geonames type 8647880 5000
+wget http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora/geonames/documents-2.json.bz2
+bzip2 -d documents-2.json.bz2
+mv documents-2.json client/benchmark/build
+gradlew -p client/benchmark run --args ' rest bulk localhost build/documents-2.json geonames type 8647880 5000'
 ```
 
-The parameters are in order:
+The parameters are all in the `'`s and are in order:
 
 * Client type: Use either "rest" or "transport"
 * Benchmark type: Use either "bulk" or "search"
 * Benchmark target host IP (the host where Elasticsearch is running)
 * full path to the file that should be bulk indexed
 * name of the index
-* name of the (sole) type in the index 
+* name of the (sole) type in the index
 * number of documents in the file
 * bulk size
 
 
-#### Bulk indexing
+#### Search
 
-Example command line parameters:
+Example invocation:
 
 ```
-rest search 192.168.2.2 geonames "{ \"query\": { \"match_phrase\": { \"name\": \"Sankt Georgen\" } } }\"" 500,1000,1100,1200
+gradlew -p client/benchmark run --args ' rest search localhost geonames {"query":{"match_phrase":{"name":"Sankt Georgen"}}} 500,1000,1100,1200'
 ```
 
 The parameters are in order:
@@ -49,5 +60,3 @@ The parameters are in order:
 * name of the index
 * a search request body (remember to escape double quotes). The `TransportClientBenchmark` uses `QueryBuilders.wrapperQuery()` internally which automatically adds a root key `query`, so it must not be present in the command line parameter.
 * A comma-separated list of target throughput rates
-
-
diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle
index 0c3238d9853..c67120c7cf5 100644
--- a/client/benchmark/build.gradle
+++ b/client/benchmark/build.gradle
@@ -18,9 +18,6 @@
  */
 
 apply plugin: 'elasticsearch.build'
-// build an uberjar with all benchmarks
-apply plugin: 'com.github.johnrengelman.shadow'
-// have the shadow plugin provide the runShadow task
 apply plugin: 'application'
 
 group = 'org.elasticsearch.client'
@@ -32,7 +29,6 @@ build.dependsOn.remove('assemble')
 archivesBaseName = 'client-benchmarks'
 mainClassName = 'org.elasticsearch.client.benchmark.BenchmarkMain'
 
-
 // never try to invoke tests on the benchmark project - there aren't any
 test.enabled = false
 

From 09e38f2f5994b45acf7cd6b7f512bc50dffca4c9 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 31 Jul 2018 16:25:53 -0700
Subject: [PATCH 253/260] Painless: Clean up PainlessMethod (#32476)

Renames and removes variables from PainlessMethod to follow the new naming
convention. Generates methodtypes at compile-time instead of using a method at run-
time. Moves write method to MethodWriter.
---
 .../java/org/elasticsearch/painless/Def.java  |  15 +-
 .../elasticsearch/painless/FunctionRef.java   | 129 +++++++++++-------
 .../org/elasticsearch/painless/Locals.java    |  42 ++++--
 .../elasticsearch/painless/MethodWriter.java  |  24 ++++
 .../lookup/PainlessLookupBuilder.java         |  35 ++---
 .../painless/lookup/PainlessMethod.java       |  67 ++-------
 .../painless/node/ECallLocal.java             |  11 +-
 .../painless/node/ECapturingFunctionRef.java  |   8 +-
 .../painless/node/EFunctionRef.java           |  14 +-
 .../elasticsearch/painless/node/ELambda.java  |  23 ++--
 .../painless/node/EListInit.java              |   2 +-
 .../elasticsearch/painless/node/EMapInit.java |   2 +-
 .../painless/node/PSubCallInvoke.java         |   8 +-
 .../painless/node/PSubListShortcut.java       |  25 ++--
 .../painless/node/PSubMapShortcut.java        |  32 ++---
 .../painless/node/PSubShortcut.java           |  24 ++--
 .../painless/node/SFunction.java              |  32 ++---
 .../elasticsearch/painless/node/SSource.java  |  12 +-
 .../painless/node/SSubEachIterable.java       |   2 +-
 .../painless/PainlessDocGenerator.java        |  32 ++---
 20 files changed, 280 insertions(+), 259 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 9be2559e67a..91c25b7cfec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -238,7 +238,7 @@ public final class Def {
          int numArguments = callSiteType.parameterCount();
          // simple case: no lambdas
          if (recipeString.isEmpty()) {
-             return lookupMethodInternal(painlessLookup, receiverClass, name, numArguments - 1).handle;
+             return lookupMethodInternal(painlessLookup, receiverClass, name, numArguments - 1).methodHandle;
          }
 
          // convert recipe string to a bitset for convenience (the code below should be refactored...)
@@ -262,7 +262,7 @@ public final class Def {
          // lookup the method with the proper arity, then we know everything (e.g. interface types of parameters).
          // based on these we can finally link any remaining lambdas that were deferred.
          PainlessMethod method = lookupMethodInternal(painlessLookup, receiverClass, name, arity);
-         MethodHandle handle = method.handle;
+         MethodHandle handle = method.methodHandle;
 
          int replaced = 0;
          upTo = 1;
@@ -281,7 +281,7 @@ public final class Def {
                      captures[capture] = callSiteType.parameterType(i + 1 + capture);
                  }
                  MethodHandle filter;
-                 Class<?> interfaceType = method.arguments.get(i - 1 - replaced);
+                 Class<?> interfaceType = method.typeParameters.get(i - 1 - replaced);
                  if (signature.charAt(0) == 'S') {
                      // the implementation is strongly typed, now that we know the interface type,
                      // we have everything.
@@ -331,10 +331,11 @@ public final class Def {
          if (interfaceMethod == null) {
              throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface");
          }
-         int arity = interfaceMethod.arguments.size();
+         int arity = interfaceMethod.typeParameters.size();
          PainlessMethod implMethod = lookupMethodInternal(painlessLookup, receiverClass, name, arity);
         return lookupReferenceInternal(painlessLookup, methodHandlesLookup, interfaceType,
-                PainlessLookupUtility.typeToCanonicalTypeName(implMethod.target), implMethod.name, receiverClass);
+                PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass),
+                implMethod.javaMethod.getName(), receiverClass);
      }
 
      /** Returns a method handle to an implementation of clazz, given method reference signature. */
@@ -349,7 +350,7 @@ public final class Def {
                  throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                          "to [" + PainlessLookupUtility.typeToCanonicalTypeName(clazz) + "], not a functional interface");
              }
-             int arity = interfaceMethod.arguments.size() + captures.length;
+             int arity = interfaceMethod.typeParameters.size() + captures.length;
              final MethodHandle handle;
              try {
                  MethodHandle accessor = methodHandlesLookup.findStaticGetter(methodHandlesLookup.lookupClass(),
@@ -360,7 +361,7 @@ public final class Def {
                  // is it a synthetic method? If we generated the method ourselves, be more helpful. It can only fail
                  // because the arity does not match the expected interface type.
                  if (call.contains("$")) {
-                     throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name +
+                     throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.javaMethod.getName() +
                                                         "] in [" + clazz + "]");
                  }
                  throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments.");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 0da91438326..d4671f05b6c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessLookup;
@@ -108,24 +109,24 @@ public class FunctionRef {
         Constructor<?> javaConstructor = delegateConstructor.javaConstructor;
         MethodType delegateMethodType = delegateConstructor.methodType;
 
-        interfaceMethodName = interfaceMethod.name;
-        factoryMethodType = MethodType.methodType(expected,
+        this.interfaceMethodName = interfaceMethod.javaMethod.getName();
+        this.factoryMethodType = MethodType.methodType(expected,
                 delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
-        interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
+        this.interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
 
-        delegateClassName = javaConstructor.getDeclaringClass().getName();
-        isDelegateInterface = false;
-        delegateInvokeType = H_NEWINVOKESPECIAL;
-        delegateMethodName = PainlessLookupUtility.CONSTRUCTOR_NAME;
+        this.delegateClassName = javaConstructor.getDeclaringClass().getName();
+        this.isDelegateInterface = false;
+        this.delegateInvokeType = H_NEWINVOKESPECIAL;
+        this.delegateMethodName = PainlessLookupUtility.CONSTRUCTOR_NAME;
         this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
 
         this.interfaceMethod = interfaceMethod;
-        delegateTypeParameters = delegateConstructor.typeParameters;
-        delegateReturnType = void.class;
+        this.delegateTypeParameters = delegateConstructor.typeParameters;
+        this.delegateReturnType = void.class;
 
-        factoryDescriptor = factoryMethodType.toMethodDescriptorString();
-        interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
-        delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString());
+        this.factoryDescriptor = factoryMethodType.toMethodDescriptorString();
+        this.interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
+        this.delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString());
     }
 
     /**
@@ -138,41 +139,63 @@ public class FunctionRef {
     public FunctionRef(Class<?> expected, PainlessMethod interfaceMethod, PainlessMethod delegateMethod, int numCaptures) {
         MethodType delegateMethodType = delegateMethod.methodType;
 
-        interfaceMethodName = interfaceMethod.name;
-        factoryMethodType = MethodType.methodType(expected,
+        this.interfaceMethodName = interfaceMethod.javaMethod.getName();
+        this.factoryMethodType = MethodType.methodType(expected,
                 delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
-        interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
+        this.interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
 
-        // the Painless$Script class can be inferred if owner is null
-        if (delegateMethod.target == null) {
-            delegateClassName = CLASS_NAME;
-            isDelegateInterface = false;
-        } else if (delegateMethod.augmentation != null) {
-            delegateClassName = delegateMethod.augmentation.getName();
-            isDelegateInterface = delegateMethod.augmentation.isInterface();
+        this.delegateClassName = delegateMethod.javaMethod.getDeclaringClass().getName();
+        this.isDelegateInterface = delegateMethod.javaMethod.getDeclaringClass().isInterface();
+
+        if (Modifier.isStatic(delegateMethod.javaMethod.getModifiers())) {
+            this.delegateInvokeType = H_INVOKESTATIC;
+        } else if (delegateMethod.javaMethod.getDeclaringClass().isInterface()) {
+            this.delegateInvokeType = H_INVOKEINTERFACE;
         } else {
-            delegateClassName = delegateMethod.target.getName();
-            isDelegateInterface = delegateMethod.target.isInterface();
+            this.delegateInvokeType = H_INVOKEVIRTUAL;
         }
 
-        if (Modifier.isStatic(delegateMethod.modifiers)) {
-            delegateInvokeType = H_INVOKESTATIC;
-        } else if (delegateMethod.target.isInterface()) {
-            delegateInvokeType = H_INVOKEINTERFACE;
-        } else {
-            delegateInvokeType = H_INVOKEVIRTUAL;
-        }
-
-        delegateMethodName = delegateMethod.name;
+        this.delegateMethodName = delegateMethod.javaMethod.getName();
         this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
 
         this.interfaceMethod = interfaceMethod;
-        delegateTypeParameters = delegateMethod.arguments;
-        delegateReturnType = delegateMethod.rtn;
+        this.delegateTypeParameters = delegateMethod.typeParameters;
+        this.delegateReturnType = delegateMethod.returnType;
 
-        factoryDescriptor = factoryMethodType.toMethodDescriptorString();
-        interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
-        delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString());
+        this.factoryDescriptor = factoryMethodType.toMethodDescriptorString();
+        this.interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
+        this.delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString());
+    }
+
+    /**
+     * Creates a new FunctionRef (already resolved)
+     * @param expected functional interface type to implement
+     * @param interfaceMethod functional interface method
+     * @param delegateMethod implementation method
+     * @param numCaptures number of captured arguments
+     */
+    public FunctionRef(Class<?> expected, PainlessMethod interfaceMethod, LocalMethod delegateMethod, int numCaptures) {
+        MethodType delegateMethodType = delegateMethod.methodType;
+
+        this.interfaceMethodName = interfaceMethod.javaMethod.getName();
+        this.factoryMethodType = MethodType.methodType(expected,
+                delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
+        this.interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
+
+        this.delegateClassName = CLASS_NAME;
+        this.isDelegateInterface = false;
+        this.delegateInvokeType = H_INVOKESTATIC;
+
+        this.delegateMethodName = delegateMethod.name;
+        this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
+
+        this.interfaceMethod = interfaceMethod;
+        this.delegateTypeParameters = delegateMethod.typeParameters;
+        this.delegateReturnType = delegateMethod.returnType;
+
+        this.factoryDescriptor = factoryMethodType.toMethodDescriptorString();
+        this.interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString());
+        this.delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString());
     }
 
     /**
@@ -181,24 +204,24 @@ public class FunctionRef {
      */
     public FunctionRef(Class<?> expected,
                        PainlessMethod interfaceMethod, String delegateMethodName, MethodType delegateMethodType, int numCaptures) {
-        interfaceMethodName = interfaceMethod.name;
-        factoryMethodType = MethodType.methodType(expected,
+        this.interfaceMethodName = interfaceMethod.javaMethod.getName();
+        this.factoryMethodType = MethodType.methodType(expected,
             delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount()));
-        interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
+        this.interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1);
 
-        delegateClassName = CLASS_NAME;
-        delegateInvokeType = H_INVOKESTATIC;
+        this.delegateClassName = CLASS_NAME;
+        this.delegateInvokeType = H_INVOKESTATIC;
         this.delegateMethodName = delegateMethodName;
         this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
-        isDelegateInterface = false;
+        this.isDelegateInterface = false;
 
         this.interfaceMethod = null;
-        delegateTypeParameters = null;
-        delegateReturnType = null;
+        this.delegateTypeParameters = null;
+        this.delegateReturnType = null;
 
-        factoryDescriptor = null;
-        interfaceType = null;
-        delegateType = null;
+        this.factoryDescriptor = null;
+        this.interfaceType = null;
+        this.delegateType = null;
     }
 
     /**
@@ -215,7 +238,7 @@ public class FunctionRef {
 
         // lookup requested constructor
         PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(painlessLookup.getJavaClassFromPainlessType(type));
-        PainlessConstructor impl = struct.constructors.get(PainlessLookupUtility.buildPainlessConstructorKey(method.arguments.size()));
+        PainlessConstructor impl = struct.constructors.get(PainlessLookupUtility.buildPainlessConstructorKey(method.typeParameters.size()));
 
         if (impl == null) {
             throw new IllegalArgumentException("Unknown reference [" + type + "::new] matching [" + expected + "]");
@@ -242,16 +265,16 @@ public class FunctionRef {
         final PainlessMethod impl;
         // look for a static impl first
         PainlessMethod staticImpl =
-                struct.staticMethods.get(PainlessLookupUtility.buildPainlessMethodKey(call, method.arguments.size()));
+                struct.staticMethods.get(PainlessLookupUtility.buildPainlessMethodKey(call, method.typeParameters.size()));
         if (staticImpl == null) {
             // otherwise a virtual impl
             final int arity;
             if (receiverCaptured) {
                 // receiver captured
-                arity = method.arguments.size();
+                arity = method.typeParameters.size();
             } else {
                 // receiver passed
-                arity = method.arguments.size() - 1;
+                arity = method.typeParameters.size() - 1;
             }
             impl = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(call, arity));
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index 804f6aa2b68..f2c7e02c637 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -22,8 +22,8 @@ package org.elasticsearch.painless;
 import org.elasticsearch.painless.ScriptClassInfo.MethodArgument;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 
+import java.lang.invoke.MethodType;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -38,6 +38,30 @@ import java.util.Set;
  */
 public final class Locals {
 
+    /**
+     * Constructs a local method key used to lookup local methods from a painless class.
+     */
+    public static String buildLocalMethodKey(String methodName, int methodArity) {
+        return methodName + "/" + methodArity;
+    }
+
+    /**
+     * Stores information about methods directly callable on the generated script class.
+     */
+    public static class LocalMethod {
+        public final String name;
+        public final Class<?> returnType;
+        public final List<Class<?>> typeParameters;
+        public final MethodType methodType;
+
+        public LocalMethod(String name, Class<?> returnType, List<Class<?>> typeParameters, MethodType methodType) {
+            this.name = name;
+            this.returnType = returnType;
+            this.typeParameters = typeParameters;
+            this.methodType = methodType;
+        }
+    }
+
     /** Reserved word: loop counter */
     public static final String LOOP   = "#loop";
     /** Reserved word: unused */
@@ -110,9 +134,9 @@ public final class Locals {
     }
 
     /** Creates a new program scope: the list of methods. It is the parent for all methods */
-    public static Locals newProgramScope(PainlessLookup painlessLookup, Collection<PainlessMethod> methods) {
+    public static Locals newProgramScope(PainlessLookup painlessLookup, Collection<LocalMethod> methods) {
         Locals locals = new Locals(null, painlessLookup, null, null);
-        for (PainlessMethod method : methods) {
+        for (LocalMethod method : methods) {
             locals.addMethod(method);
         }
         return locals;
@@ -143,8 +167,8 @@ public final class Locals {
     }
 
     /** Looks up a method. Returns null if the method does not exist. */
-    public PainlessMethod getMethod(String key) {
-        PainlessMethod method = lookupMethod(key);
+    public LocalMethod getMethod(String key) {
+        LocalMethod method = lookupMethod(key);
         if (method != null) {
             return method;
         }
@@ -199,7 +223,7 @@ public final class Locals {
     // variable name -> variable
     private Map<String,Variable> variables;
     // method name+arity -> methods
-    private Map<String,PainlessMethod> methods;
+    private Map<String,LocalMethod> methods;
 
     /**
      * Create a new Locals
@@ -237,7 +261,7 @@ public final class Locals {
     }
 
     /** Looks up a method at this scope only. Returns null if the method does not exist. */
-    private PainlessMethod lookupMethod(String key) {
+    private LocalMethod lookupMethod(String key) {
         if (methods == null) {
             return null;
         }
@@ -256,11 +280,11 @@ public final class Locals {
         return variable;
     }
 
-    private void addMethod(PainlessMethod method) {
+    private void addMethod(LocalMethod method) {
         if (methods == null) {
             methods = new HashMap<>();
         }
-        methods.put(PainlessLookupUtility.buildPainlessMethodKey(method.name, method.arguments.size()), method);
+        methods.put(buildLocalMethodKey(method.name, method.typeParameters.size()), method);
         // TODO: check result
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index c339e7bfb26..72435562a3b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.lookup.def;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.Label;
@@ -28,6 +29,7 @@ import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.GeneratorAdapter;
 import org.objectweb.asm.commons.Method;
 
+import java.lang.reflect.Modifier;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -415,4 +417,26 @@ public final class MethodWriter extends GeneratorAdapter {
         System.arraycopy(params, 0, args, 2, params.length);
         invokeDynamic(name, methodType.getDescriptor(), DEF_BOOTSTRAP_HANDLE, args);
     }
+
+    public void invokeMethodCall(PainlessMethod painlessMethod) {
+        Type type = Type.getType(painlessMethod.javaMethod.getDeclaringClass());
+        Method method = Method.getMethod(painlessMethod.javaMethod);
+
+        if (Modifier.isStatic(painlessMethod.javaMethod.getModifiers())) {
+            // invokeStatic assumes that the owner class is not an interface, so this is a
+            // special case for interfaces where the interface method boolean needs to be set to
+            // true to reference the appropriate class constant when calling a static interface
+            // method since java 8 did not check, but java 9 and 10 do
+            if (painlessMethod.javaMethod.getDeclaringClass().isInterface()) {
+                visitMethodInsn(Opcodes.INVOKESTATIC, type.getInternalName(),
+                        painlessMethod.javaMethod.getName(), painlessMethod.methodType.toMethodDescriptorString(), true);
+            } else {
+                invokeStatic(type, method);
+            }
+        } else if (painlessMethod.javaMethod.getDeclaringClass().isInterface()) {
+            invokeInterface(type, method);
+        } else {
+            invokeVirtual(type, method);
+        }
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index b0a11dea58a..a4dbe1006d6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -540,7 +540,6 @@ public final class PainlessLookupBuilder {
             PainlessMethod painlessMethod = painlessClassBuilder.staticMethods.get(painlessMethodKey);
 
             if (painlessMethod == null) {
-                org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod);
                 MethodHandle methodHandle;
 
                 try {
@@ -554,19 +553,17 @@ public final class PainlessLookupBuilder {
 
                 painlessMethod = painlessMethodCache.computeIfAbsent(
                         new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
-                        key -> new PainlessMethod(methodName, targetClass, null, returnType,
-                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle, methodType));
+                        key -> new PainlessMethod(javaMethod, targetClass, returnType, typeParameters, methodHandle, methodType));
 
                 painlessClassBuilder.staticMethods.put(painlessMethodKey, painlessMethod);
-            } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
-                    painlessMethod.arguments.equals(typeParameters)) == false) {
+            } else if (painlessMethod.returnType == returnType && painlessMethod.typeParameters.equals(typeParameters) == false) {
                 throw new IllegalArgumentException("cannot have static methods " +
                         "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
                         "[" + typeToCanonicalTypeName(returnType) + "], " +
                         typesToCanonicalTypeNames(typeParameters) + "] and " +
                         "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
-                        "[" + typeToCanonicalTypeName(painlessMethod.rtn) + "], " +
-                        typesToCanonicalTypeNames(painlessMethod.arguments) + "] " +
+                        "[" + typeToCanonicalTypeName(painlessMethod.returnType) + "], " +
+                        typesToCanonicalTypeNames(painlessMethod.typeParameters) + "] " +
                         "with the same arity and different return type or type parameters");
             }
         } else {
@@ -597,19 +594,17 @@ public final class PainlessLookupBuilder {
 
                 painlessMethod = painlessMethodCache.computeIfAbsent(
                         new PainlessMethodCacheKey(targetClass, methodName, typeParameters),
-                        key -> new PainlessMethod(methodName, targetClass, augmentedClass, returnType,
-                                typeParameters, asmMethod, javaMethod.getModifiers(), methodHandle, methodType));
+                        key -> new PainlessMethod(javaMethod, targetClass, returnType, typeParameters, methodHandle, methodType));
 
                 painlessClassBuilder.methods.put(painlessMethodKey, painlessMethod);
-            } else if ((painlessMethod.name.equals(methodName) && painlessMethod.rtn == returnType &&
-                    painlessMethod.arguments.equals(typeParameters)) == false) {
+            } else if (painlessMethod.returnType == returnType && painlessMethod.typeParameters.equals(typeParameters) == false) {
                 throw new IllegalArgumentException("cannot have methods " +
                         "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
                         "[" + typeToCanonicalTypeName(returnType) + "], " +
                         typesToCanonicalTypeNames(typeParameters) + "] and " +
                         "[[" + targetCanonicalClassName + "], [" + methodName + "], " +
-                        "[" + typeToCanonicalTypeName(painlessMethod.rtn) + "], " +
-                        typesToCanonicalTypeNames(painlessMethod.arguments) + "] " +
+                        "[" + typeToCanonicalTypeName(painlessMethod.returnType) + "], " +
+                        typesToCanonicalTypeNames(painlessMethod.typeParameters) + "] " +
                         "with the same arity and different return type or type parameters");
             }
         }
@@ -806,8 +801,8 @@ public final class PainlessLookupBuilder {
             PainlessMethod newPainlessMethod = painlessMethodEntry.getValue();
             PainlessMethod existingPainlessMethod = targetPainlessClassBuilder.methods.get(painlessMethodKey);
 
-            if (existingPainlessMethod == null || existingPainlessMethod.target != newPainlessMethod.target &&
-                    existingPainlessMethod.target.isAssignableFrom(newPainlessMethod.target)) {
+            if (existingPainlessMethod == null || existingPainlessMethod.targetClass != newPainlessMethod.targetClass &&
+                    existingPainlessMethod.targetClass.isAssignableFrom(newPainlessMethod.targetClass)) {
                 targetPainlessClassBuilder.methods.put(painlessMethodKey, newPainlessMethod);
             }
         }
@@ -832,21 +827,21 @@ public final class PainlessLookupBuilder {
 
     private void cacheRuntimeHandles(PainlessClassBuilder painlessClassBuilder) {
         for (PainlessMethod painlessMethod : painlessClassBuilder.methods.values()) {
-            String methodName = painlessMethod.name;
-            int typeParametersSize = painlessMethod.arguments.size();
+            String methodName = painlessMethod.javaMethod.getName();
+            int typeParametersSize = painlessMethod.typeParameters.size();
 
             if (typeParametersSize == 0 && methodName.startsWith("get") && methodName.length() > 3 &&
                     Character.isUpperCase(methodName.charAt(3))) {
                 painlessClassBuilder.getterMethodHandles.putIfAbsent(
-                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.handle);
+                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.methodHandle);
             } else if (typeParametersSize == 0 && methodName.startsWith("is") && methodName.length() > 2 &&
                     Character.isUpperCase(methodName.charAt(2))) {
                 painlessClassBuilder.getterMethodHandles.putIfAbsent(
-                        Character.toLowerCase(methodName.charAt(2)) + methodName.substring(3), painlessMethod.handle);
+                        Character.toLowerCase(methodName.charAt(2)) + methodName.substring(3), painlessMethod.methodHandle);
             } else if (typeParametersSize == 1 && methodName.startsWith("set") && methodName.length() > 3 &&
                     Character.isUpperCase(methodName.charAt(3))) {
                 painlessClassBuilder.setterMethodHandles.putIfAbsent(
-                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.handle);
+                        Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4), painlessMethod.methodHandle);
             }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
index 904cf06907f..9dd143a4028 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java
@@ -19,67 +19,28 @@
 
 package org.elasticsearch.painless.lookup;
 
-import org.elasticsearch.painless.MethodWriter;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.Type;
-
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodType;
-import java.lang.reflect.Modifier;
+import java.lang.reflect.Method;
 import java.util.Collections;
 import java.util.List;
 
 public class PainlessMethod {
-    public final String name;
-    public final Class<?> target;
-    public final Class<?> augmentation;
-    public final Class<?> rtn;
-    public final List<Class<?>> arguments;
-    public final org.objectweb.asm.commons.Method method;
-    public final int modifiers;
-    public final MethodHandle handle;
+    public final Method javaMethod;
+    public final Class<?> targetClass;
+    public final Class<?> returnType;
+    public final List<Class<?>> typeParameters;
+    public final MethodHandle methodHandle;
     public final MethodType methodType;
 
-    public PainlessMethod(String name, Class<?> target, Class<?> augmentation, Class<?> rtn, List<Class<?>> arguments,
-                          org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle, MethodType methodType) {
-        this.name = name;
-        this.augmentation = augmentation;
-        this.target = target;
-        this.rtn = rtn;
-        this.arguments = Collections.unmodifiableList(arguments);
-        this.method = method;
-        this.modifiers = modifiers;
-        this.handle = handle;
+    public PainlessMethod(Method javaMethod, Class<?> targetClass, Class<?> returnType, List<Class<?>> typeParameters,
+            MethodHandle methodHandle, MethodType methodType) {
+
+        this.javaMethod = javaMethod;
+        this.targetClass = targetClass;
+        this.returnType = returnType;
+        this.typeParameters = Collections.unmodifiableList(typeParameters);
+        this.methodHandle = methodHandle;
         this.methodType = methodType;
     }
-
-    public void write(MethodWriter writer) {
-        final org.objectweb.asm.Type type;
-        final Class<?> clazz;
-        if (augmentation != null) {
-            assert Modifier.isStatic(modifiers);
-            clazz = augmentation;
-            type = org.objectweb.asm.Type.getType(augmentation);
-        } else {
-            clazz = target;
-            type = Type.getType(target);
-        }
-
-        if (Modifier.isStatic(modifiers)) {
-            // invokeStatic assumes that the owner class is not an interface, so this is a
-            // special case for interfaces where the interface method boolean needs to be set to
-            // true to reference the appropriate class constant when calling a static interface
-            // method since java 8 did not check, but java 9 and 10 do
-            if (Modifier.isInterface(clazz.getModifiers())) {
-                writer.visitMethodInsn(Opcodes.INVOKESTATIC,
-                        type.getInternalName(), name, methodType.toMethodDescriptorString(), true);
-            } else {
-                writer.invokeStatic(type, method);
-            }
-        } else if (Modifier.isInterface(clazz.getModifiers())) {
-            writer.invokeInterface(type, method);
-        } else {
-            writer.invokeVirtual(type, method);
-        }
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
index 098c75386e1..7605a0c9f7f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java
@@ -21,10 +21,11 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
+import org.objectweb.asm.commons.Method;
 
 import java.util.List;
 import java.util.Objects;
@@ -40,7 +41,7 @@ public final class ECallLocal extends AExpression {
     private final String name;
     private final List<AExpression> arguments;
 
-    private PainlessMethod method = null;
+    private LocalMethod method = null;
 
     public ECallLocal(Location location, String name, List<AExpression> arguments) {
         super(location);
@@ -68,14 +69,14 @@ public final class ECallLocal extends AExpression {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = method.arguments.get(argument);
+            expression.expected = method.typeParameters.get(argument);
             expression.internal = true;
             expression.analyze(locals);
             arguments.set(argument, expression.cast(locals));
         }
 
         statement = true;
-        actual = method.rtn;
+        actual = method.returnType;
     }
 
     @Override
@@ -86,7 +87,7 @@ public final class ECallLocal extends AExpression {
             argument.write(writer, globals);
         }
 
-        writer.invokeStatic(CLASS_TYPE, method.method);
+        writer.invokeStatic(CLASS_TYPE, new Method(method.name, method.methodType.toMethodDescriptorString()));
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index a4f7b130045..e78b3c67210 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -81,14 +81,14 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda
                             PainlessLookupUtility.typeToCanonicalTypeName(captured.clazz), call, 1);
 
                     // check casts between the interface method and the delegate method are legal
-                    for (int i = 0; i < ref.interfaceMethod.arguments.size(); ++i) {
-                        Class<?> from = ref.interfaceMethod.arguments.get(i);
+                    for (int i = 0; i < ref.interfaceMethod.typeParameters.size(); ++i) {
+                        Class<?> from = ref.interfaceMethod.typeParameters.get(i);
                         Class<?> to = ref.delegateTypeParameters.get(i);
                         AnalyzerCaster.getLegalCast(location, from, to, false, true);
                     }
 
-                    if (ref.interfaceMethod.rtn != void.class) {
-                        AnalyzerCaster.getLegalCast(location, ref.delegateReturnType, ref.interfaceMethod.rtn, false, true);
+                    if (ref.interfaceMethod.returnType != void.class) {
+                        AnalyzerCaster.getLegalCast(location, ref.delegateReturnType, ref.interfaceMethod.returnType, false, true);
                     }
                 } catch (IllegalArgumentException e) {
                     throw createError(e);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 23fb8a4180e..782991e2958 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
@@ -70,8 +71,7 @@ public final class EFunctionRef extends AExpression implements ILambda {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                 "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface");
                     }
-                    PainlessMethod delegateMethod =
-                            locals.getMethod(PainlessLookupUtility.buildPainlessMethodKey(call, interfaceMethod.arguments.size()));
+                    LocalMethod delegateMethod = locals.getMethod(Locals.buildLocalMethodKey(call, interfaceMethod.typeParameters.size()));
                     if (delegateMethod == null) {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                 "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], function not found");
@@ -79,14 +79,14 @@ public final class EFunctionRef extends AExpression implements ILambda {
                     ref = new FunctionRef(expected, interfaceMethod, delegateMethod, 0);
 
                     // check casts between the interface method and the delegate method are legal
-                    for (int i = 0; i < interfaceMethod.arguments.size(); ++i) {
-                        Class<?> from = interfaceMethod.arguments.get(i);
-                        Class<?> to = delegateMethod.arguments.get(i);
+                    for (int i = 0; i < interfaceMethod.typeParameters.size(); ++i) {
+                        Class<?> from = interfaceMethod.typeParameters.get(i);
+                        Class<?> to = delegateMethod.typeParameters.get(i);
                         AnalyzerCaster.getLegalCast(location, from, to, false, true);
                     }
 
-                    if (interfaceMethod.rtn != void.class) {
-                        AnalyzerCaster.getLegalCast(location, delegateMethod.rtn, interfaceMethod.rtn, false, true);
+                    if (interfaceMethod.returnType != void.class) {
+                        AnalyzerCaster.getLegalCast(location, delegateMethod.returnType, interfaceMethod.returnType, false, true);
                     }
                 } else {
                     // whitelist lookup
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index ab1442be805..6fc4a3a6480 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -126,21 +127,21 @@ public final class ELambda extends AExpression implements ILambda {
                         "[" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"));
             }
             // check arity before we manipulate parameters
-            if (interfaceMethod.arguments.size() != paramTypeStrs.size())
-                throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name +
+            if (interfaceMethod.typeParameters.size() != paramTypeStrs.size())
+                throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.javaMethod.getName() +
                         "] in [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "]");
             // for method invocation, its allowed to ignore the return value
-            if (interfaceMethod.rtn == void.class) {
+            if (interfaceMethod.returnType == void.class) {
                 returnType = def.class;
             } else {
-                returnType = interfaceMethod.rtn;
+                returnType = interfaceMethod.returnType;
             }
             // replace any null types with the actual type
             actualParamTypeStrs = new ArrayList<>(paramTypeStrs.size());
             for (int i = 0; i < paramTypeStrs.size(); i++) {
                 String paramType = paramTypeStrs.get(i);
                 if (paramType == null) {
-                    actualParamTypeStrs.add(PainlessLookupUtility.typeToCanonicalTypeName(interfaceMethod.arguments.get(i)));
+                    actualParamTypeStrs.add(PainlessLookupUtility.typeToCanonicalTypeName(interfaceMethod.typeParameters.get(i)));
                 } else {
                     actualParamTypeStrs.add(paramType);
                 }
@@ -183,20 +184,22 @@ public final class ELambda extends AExpression implements ILambda {
         } else {
             defPointer = null;
             try {
-                ref = new FunctionRef(expected, interfaceMethod, desugared.method, captures.size());
+                LocalMethod localMethod =
+                        new LocalMethod(desugared.name, desugared.returnType, desugared.typeParameters, desugared.methodType);
+                ref = new FunctionRef(expected, interfaceMethod, localMethod, captures.size());
             } catch (IllegalArgumentException e) {
                 throw createError(e);
             }
 
             // check casts between the interface method and the delegate method are legal
-            for (int i = 0; i < interfaceMethod.arguments.size(); ++i) {
-                Class<?> from = interfaceMethod.arguments.get(i);
+            for (int i = 0; i < interfaceMethod.typeParameters.size(); ++i) {
+                Class<?> from = interfaceMethod.typeParameters.get(i);
                 Class<?> to = desugared.parameters.get(i + captures.size()).clazz;
                 AnalyzerCaster.getLegalCast(location, from, to, false, true);
             }
 
-            if (interfaceMethod.rtn != void.class) {
-                AnalyzerCaster.getLegalCast(location, desugared.rtnType, interfaceMethod.rtn, false, true);
+            if (interfaceMethod.returnType != void.class) {
+                AnalyzerCaster.getLegalCast(location, desugared.returnType, interfaceMethod.returnType, false, true);
             }
 
             actual = expected;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index bd4bea95ea0..01a4878266e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -100,7 +100,7 @@ public final class EListInit extends AExpression {
         for (AExpression value : values) {
             writer.dup();
             value.write(writer, globals);
-            method.write(writer);
+            writer.invokeMethodCall(method);
             writer.pop();
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index 5bef62a4b18..73afe7f0dc5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -123,7 +123,7 @@ public final class EMapInit extends AExpression {
             writer.dup();
             key.write(writer, globals);
             value.write(writer, globals);
-            method.write(writer);
+            writer.invokeMethodCall(method);
             writer.pop();
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 237efa61ffa..fe2ae52603b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -56,14 +56,14 @@ final class PSubCallInvoke extends AExpression {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = method.arguments.get(argument);
+            expression.expected = method.typeParameters.get(argument);
             expression.internal = true;
             expression.analyze(locals);
             arguments.set(argument, expression.cast(locals));
         }
 
         statement = true;
-        actual = method.rtn;
+        actual = method.returnType;
     }
 
     @Override
@@ -78,11 +78,11 @@ final class PSubCallInvoke extends AExpression {
             argument.write(writer, globals);
         }
 
-        method.write(writer);
+        writer.invokeMethodCall(method);
     }
 
     @Override
     public String toString() {
-        return singleLineToStringWithOptionalArgs(arguments, prefix, method.name);
+        return singleLineToStringWithOptionalArgs(arguments, prefix, method.javaMethod.getName());
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
index 509aad64153..0738f55c2cf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java
@@ -62,17 +62,17 @@ final class PSubListShortcut extends AStoreable {
         getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1));
         setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("set", 2));
 
-        if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1 ||
-            getter.arguments.get(0) != int.class)) {
+        if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1 ||
+            getter.typeParameters.get(0) != int.class)) {
             throw createError(new IllegalArgumentException("Illegal list get shortcut for type [" + canonicalClassName + "]."));
         }
 
-        if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0) != int.class)) {
+        if (setter != null && (setter.typeParameters.size() != 2 || setter.typeParameters.get(0) != int.class)) {
             throw createError(new IllegalArgumentException("Illegal list set shortcut for type [" + canonicalClassName + "]."));
         }
 
-        if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
-            || !getter.rtn.equals(setter.arguments.get(1)))) {
+        if (getter != null && setter != null && (!getter.typeParameters.get(0).equals(setter.typeParameters.get(0))
+            || !getter.returnType.equals(setter.typeParameters.get(1)))) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
@@ -81,7 +81,7 @@ final class PSubListShortcut extends AStoreable {
             index.analyze(locals);
             index = index.cast(locals);
 
-            actual = setter != null ? setter.arguments.get(1) : getter.rtn;
+            actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + canonicalClassName + "]."));
         }
@@ -119,21 +119,18 @@ final class PSubListShortcut extends AStoreable {
     @Override
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
+        writer.invokeMethodCall(getter);
 
-        getter.write(writer);
-
-        if (getter.rtn == getter.handle.type().returnType()) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (getter.returnType == getter.javaMethod.getReturnType()) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
     @Override
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
-
-        setter.write(writer);
-
-        writer.writePop(MethodWriter.getType(setter.rtn).getSize());
+        writer.invokeMethodCall(setter);
+        writer.writePop(MethodWriter.getType(setter.returnType).getSize());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
index 2d7f2250c6c..04ccbc9f534 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java
@@ -61,25 +61,25 @@ final class PSubMapShortcut extends AStoreable {
         getter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("get", 1));
         setter = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey("put", 2));
 
-        if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1)) {
+        if (getter != null && (getter.returnType == void.class || getter.typeParameters.size() != 1)) {
             throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + canonicalClassName + "]."));
         }
 
-        if (setter != null && setter.arguments.size() != 2) {
+        if (setter != null && setter.typeParameters.size() != 2) {
             throw createError(new IllegalArgumentException("Illegal map set shortcut for type [" + canonicalClassName + "]."));
         }
 
-        if (getter != null && setter != null &&
-            (!getter.arguments.get(0).equals(setter.arguments.get(0)) || !getter.rtn.equals(setter.arguments.get(1)))) {
+        if (getter != null && setter != null && (!getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) ||
+                !getter.returnType.equals(setter.typeParameters.get(1)))) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
         if ((read || write) && (!read || getter != null) && (!write || setter != null)) {
-            index.expected = setter != null ? setter.arguments.get(0) : getter.arguments.get(0);
+            index.expected = setter != null ? setter.typeParameters.get(0) : getter.typeParameters.get(0);
             index.analyze(locals);
             index = index.cast(locals);
 
-            actual = setter != null ? setter.arguments.get(1) : getter.rtn;
+            actual = setter != null ? setter.typeParameters.get(1) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + canonicalClassName + "]."));
         }
@@ -90,11 +90,10 @@ final class PSubMapShortcut extends AStoreable {
         index.write(writer, globals);
 
         writer.writeDebugInfo(location);
+        writer.invokeMethodCall(getter);
 
-        getter.write(writer);
-
-        if (getter.rtn != getter.handle.type().returnType()) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (getter.returnType != getter.javaMethod.getReturnType()) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
@@ -121,21 +120,18 @@ final class PSubMapShortcut extends AStoreable {
     @Override
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
+        writer.invokeMethodCall(getter);
 
-        getter.write(writer);
-
-        if (getter.rtn != getter.handle.type().returnType()) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (getter.returnType != getter.javaMethod.getReturnType()) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
     @Override
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
-
-        setter.write(writer);
-
-        writer.writePop(MethodWriter.getType(setter.rtn).getSize());
+        writer.invokeMethodCall(setter);
+        writer.writePop(MethodWriter.getType(setter.returnType).getSize());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
index eb5668c554c..16975660477 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java
@@ -53,22 +53,22 @@ final class PSubShortcut extends AStoreable {
 
     @Override
     void analyze(Locals locals) {
-        if (getter != null && (getter.rtn == void.class || !getter.arguments.isEmpty())) {
+        if (getter != null && (getter.returnType == void.class || !getter.typeParameters.isEmpty())) {
             throw createError(new IllegalArgumentException(
                 "Illegal get shortcut on field [" + value + "] for type [" + type + "]."));
         }
 
-        if (setter != null && (setter.rtn != void.class || setter.arguments.size() != 1)) {
+        if (setter != null && (setter.returnType != void.class || setter.typeParameters.size() != 1)) {
             throw createError(new IllegalArgumentException(
                 "Illegal set shortcut on field [" + value + "] for type [" + type + "]."));
         }
 
-        if (getter != null && setter != null && setter.arguments.get(0) != getter.rtn) {
+        if (getter != null && setter != null && setter.typeParameters.get(0) != getter.returnType) {
             throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
         if ((getter != null || setter != null) && (!read || getter != null) && (!write || setter != null)) {
-            actual = setter != null ? setter.arguments.get(0) : getter.rtn;
+            actual = setter != null ? setter.typeParameters.get(0) : getter.returnType;
         } else {
             throw createError(new IllegalArgumentException("Illegal shortcut on field [" + value + "] for type [" + type + "]."));
         }
@@ -78,10 +78,10 @@ final class PSubShortcut extends AStoreable {
     void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        getter.write(writer);
+        writer.invokeMethodCall(getter);
 
-        if (!getter.rtn.equals(getter.handle.type().returnType())) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (!getter.returnType.equals(getter.javaMethod.getReturnType())) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
@@ -109,10 +109,10 @@ final class PSubShortcut extends AStoreable {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        getter.write(writer);
+        writer.invokeMethodCall(getter);
 
-        if (getter.rtn != getter.handle.type().returnType()) {
-            writer.checkCast(MethodWriter.getType(getter.rtn));
+        if (getter.returnType != getter.javaMethod.getReturnType()) {
+            writer.checkCast(MethodWriter.getType(getter.returnType));
         }
     }
 
@@ -120,9 +120,9 @@ final class PSubShortcut extends AStoreable {
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        setter.write(writer);
+        writer.invokeMethodCall(setter);
 
-        writer.writePop(MethodWriter.getType(setter.rtn).getSize());
+        writer.writePop(MethodWriter.getType(setter.returnType).getSize());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 600ca95504a..d61a424f83d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -31,14 +31,12 @@ import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.node.SSource.Reserved;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 
 import java.lang.invoke.MethodType;
-import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -92,9 +90,12 @@ public final class SFunction extends AStatement {
     private final List<AStatement> statements;
     public final boolean synthetic;
 
-    Class<?> rtnType = null;
+    Class<?> returnType;
+    List<Class<?>> typeParameters;
+    MethodType methodType;
+
+    org.objectweb.asm.commons.Method method;
     List<Parameter> parameters = new ArrayList<>();
-    PainlessMethod method = null;
 
     private Variable loop = null;
 
@@ -120,7 +121,7 @@ public final class SFunction extends AStatement {
 
     void generateSignature(PainlessLookup painlessLookup) {
         try {
-            rtnType = painlessLookup.getJavaClassFromPainlessType(rtnTypeStr);
+            returnType = painlessLookup.getJavaClassFromPainlessType(rtnTypeStr);
         } catch (IllegalArgumentException exception) {
             throw createError(new IllegalArgumentException("Illegal return type [" + rtnTypeStr + "] for function [" + name + "]."));
         }
@@ -145,11 +146,10 @@ public final class SFunction extends AStatement {
             }
         }
 
-        int modifiers = Modifier.STATIC | Modifier.PRIVATE;
-        org.objectweb.asm.commons.Method method = new org.objectweb.asm.commons.Method(name, MethodType.methodType(
-                PainlessLookupUtility.typeToJavaType(rtnType), paramClasses).toMethodDescriptorString());
-        MethodType methodType = MethodType.methodType(PainlessLookupUtility.typeToJavaType(rtnType), paramClasses);
-        this.method = new PainlessMethod(name, null, null, rtnType, paramTypes, method, modifiers, null, methodType);
+        typeParameters = paramTypes;
+        methodType = MethodType.methodType(PainlessLookupUtility.typeToJavaType(returnType), paramClasses);
+        method = new org.objectweb.asm.commons.Method(name, MethodType.methodType(
+                PainlessLookupUtility.typeToJavaType(returnType), paramClasses).toMethodDescriptorString());
     }
 
     @Override
@@ -177,7 +177,7 @@ public final class SFunction extends AStatement {
             allEscape = statement.allEscape;
         }
 
-        if (!methodEscape && rtnType != void.class) {
+        if (!methodEscape && returnType != void.class) {
             throw createError(new IllegalArgumentException("Not all paths provide a return value for method [" + name + "]."));
         }
 
@@ -192,7 +192,7 @@ public final class SFunction extends AStatement {
         if (synthetic) {
             access |= Opcodes.ACC_SYNTHETIC;
         }
-        final MethodWriter function = new MethodWriter(access, method.method, writer, globals.getStatements(), settings);
+        final MethodWriter function = new MethodWriter(access, method, writer, globals.getStatements(), settings);
         function.visitCode();
         write(function, globals);
         function.endMethod();
@@ -212,7 +212,7 @@ public final class SFunction extends AStatement {
         }
 
         if (!methodEscape) {
-            if (rtnType == void.class) {
+            if (returnType == void.class) {
                 function.returnValue();
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
@@ -225,11 +225,7 @@ public final class SFunction extends AStatement {
     }
 
     private void initializeConstant(MethodWriter writer) {
-        final Handle handle = new Handle(Opcodes.H_INVOKESTATIC,
-                CLASS_TYPE.getInternalName(),
-                name,
-                method.method.getDescriptor(),
-                false);
+        final Handle handle = new Handle(Opcodes.H_INVOKESTATIC, CLASS_TYPE.getInternalName(), name, method.getDescriptor(), false);
         writer.push(handle);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index c354e78a961..fe735c0db31 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Constant;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.LocalMethod;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -30,8 +31,6 @@ import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.SimpleChecksAdapter;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
 import org.elasticsearch.painless.node.SFunction.FunctionReserved;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
@@ -165,14 +164,15 @@ public final class SSource extends AStatement {
     }
 
     public void analyze(PainlessLookup painlessLookup) {
-        Map<String, PainlessMethod> methods = new HashMap<>();
+        Map<String, LocalMethod> methods = new HashMap<>();
 
         for (SFunction function : functions) {
             function.generateSignature(painlessLookup);
 
-            String key = PainlessLookupUtility.buildPainlessMethodKey(function.name, function.parameters.size());
+            String key = Locals.buildLocalMethodKey(function.name, function.parameters.size());
 
-            if (methods.put(key, function.method) != null) {
+            if (methods.put(key,
+                    new LocalMethod(function.name, function.returnType, function.typeParameters, function.methodType)) != null) {
                 throw createError(new IllegalArgumentException("Duplicate functions with name [" + function.name + "]."));
             }
         }
@@ -184,7 +184,7 @@ public final class SSource extends AStatement {
     void analyze(Locals program) {
         for (SFunction function : functions) {
             Locals functionLocals =
-                Locals.newFunctionScope(program, function.rtnType, function.parameters, function.reserved.getMaxLoopCounter());
+                Locals.newFunctionScope(program, function.returnType, function.parameters, function.reserved.getMaxLoopCounter());
             function.analyze(functionLocals);
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 12e3154eb56..5450f690f6c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -99,7 +99,7 @@ final class SSubEachIterable extends AStatement {
                     .getMethodType(org.objectweb.asm.Type.getType(Iterator.class), org.objectweb.asm.Type.getType(Object.class));
             writer.invokeDefCall("iterator", methodType, DefBootstrap.ITERATOR);
         } else {
-            method.write(writer);
+            writer.invokeMethodCall(method);
         }
 
         writer.visitVarInsn(MethodWriter.getType(iterator.clazz).getOpcode(Opcodes.ISTORE), iterator.getSlot());
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
index 65429154301..1f8410092a0 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -57,8 +57,8 @@ public class PainlessDocGenerator {
     private static final PainlessLookup PAINLESS_LOOKUP = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
     private static final Logger logger = ESLoggerFactory.getLogger(PainlessDocGenerator.class);
     private static final Comparator<PainlessField> FIELD_NAME = comparing(f -> f.name);
-    private static final Comparator<PainlessMethod> METHOD_NAME = comparing(m -> m.name);
-    private static final Comparator<PainlessMethod> METHOD_NUMBER_OF_PARAMS = comparing(m -> m.arguments.size());
+    private static final Comparator<PainlessMethod> METHOD_NAME = comparing(m -> m.javaMethod.getName());
+    private static final Comparator<PainlessMethod> METHOD_NUMBER_OF_PARAMS = comparing(m -> m.typeParameters.size());
     private static final Comparator<PainlessConstructor> CONSTRUCTOR_NUMBER_OF_PARAMS = comparing(m -> m.typeParameters.size());
 
     public static void main(String[] args) throws IOException {
@@ -114,10 +114,10 @@ public class PainlessDocGenerator {
                     struct.constructors.values().stream().sorted(CONSTRUCTOR_NUMBER_OF_PARAMS).forEach(documentConstructor);
                     Map<String, Class<?>> inherited = new TreeMap<>();
                     struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(METHOD_NUMBER_OF_PARAMS)).forEach(method -> {
-                        if (method.target == clazz) {
+                        if (method.targetClass == clazz) {
                             documentMethod(typeStream, method);
                         } else {
-                            inherited.put(canonicalClassName, method.target);
+                            inherited.put(canonicalClassName, method.targetClass);
                         }
                     });
 
@@ -212,11 +212,11 @@ public class PainlessDocGenerator {
         emitAnchor(stream, method);
         stream.print("]]");
 
-        if (null == method.augmentation && Modifier.isStatic(method.modifiers)) {
+        if (method.targetClass == method.javaMethod.getDeclaringClass() && Modifier.isStatic(method.javaMethod.getModifiers())) {
             stream.print("static ");
         }
 
-        emitType(stream, method.rtn);
+        emitType(stream, method.returnType);
         stream.print(' ');
 
         String javadocRoot = javadocRoot(method);
@@ -227,7 +227,7 @@ public class PainlessDocGenerator {
 
         stream.print("](");
         boolean first = true;
-        for (Class<?> arg : method.arguments) {
+        for (Class<?> arg : method.typeParameters) {
             if (first) {
                 first = false;
             } else {
@@ -269,11 +269,11 @@ public class PainlessDocGenerator {
      * Anchor text for a {@link PainlessMethod}.
      */
     private static void emitAnchor(PrintStream stream, PainlessMethod method) {
-        emitAnchor(stream, method.target);
+        emitAnchor(stream, method.targetClass);
         stream.print('-');
         stream.print(methodName(method));
         stream.print('-');
-        stream.print(method.arguments.size());
+        stream.print(method.typeParameters.size());
     }
 
     /**
@@ -290,7 +290,7 @@ public class PainlessDocGenerator {
     }
 
     private static String methodName(PainlessMethod method) {
-        return PainlessLookupUtility.typeToCanonicalTypeName(method.target);
+        return PainlessLookupUtility.typeToCanonicalTypeName(method.targetClass);
     }
 
     /**
@@ -359,16 +359,16 @@ public class PainlessDocGenerator {
         stream.print("link:{");
         stream.print(root);
         stream.print("-javadoc}/");
-        stream.print(classUrlPath(method.augmentation != null ? method.augmentation : method.target));
+        stream.print(classUrlPath(method.javaMethod.getDeclaringClass()));
         stream.print(".html#");
         stream.print(methodName(method));
         stream.print("%2D");
         boolean first = true;
-        if (method.augmentation != null) {
+        if (method.targetClass != method.javaMethod.getDeclaringClass()) {
             first = false;
-            stream.print(method.target.getName());
+            stream.print(method.javaMethod.getDeclaringClass().getName());
         }
-        for (Class<?> clazz: method.arguments) {
+        for (Class<?> clazz: method.typeParameters) {
             if (first) {
                 first = false;
             } else {
@@ -400,10 +400,10 @@ public class PainlessDocGenerator {
      * Pick the javadoc root for a {@link PainlessMethod}.
      */
     private static String javadocRoot(PainlessMethod method) {
-        if (method.augmentation != null) {
+        if (method.targetClass != method.javaMethod.getDeclaringClass()) {
             return "painless";
         }
-        return javadocRoot(method.target);
+        return javadocRoot(method.targetClass);
     }
 
     /**

From cc6d6cae7cee3620ff72fdca909515b0c1916553 Mon Sep 17 00:00:00 2001
From: Lisa Cawley <lcawley@elastic.co>
Date: Tue, 31 Jul 2018 16:28:09 -0700
Subject: [PATCH 254/260] [DOCS] Small fixes in rule configuration page
 (#32516)

---
 .../docs/en/ml/detector-custom-rules.asciidoc | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/x-pack/docs/en/ml/detector-custom-rules.asciidoc b/x-pack/docs/en/ml/detector-custom-rules.asciidoc
index 18d516fae2d..e0ca3605284 100644
--- a/x-pack/docs/en/ml/detector-custom-rules.asciidoc
+++ b/x-pack/docs/en/ml/detector-custom-rules.asciidoc
@@ -16,7 +16,7 @@ Let us see how those can be configured by examples.
 
 ==== Specifying rule scope
 
-Let us assume we are configuring a job in order to DNS data exfiltration.
+Let us assume we are configuring a job in order to detect DNS data exfiltration.
 Our data contain fields "subdomain" and "highest_registered_domain".
 We can use a detector that looks like `high_info_content(subdomain) over highest_registered_domain`.
 If we run such a job it is possible that we discover a lot of anomalies on 
@@ -25,8 +25,8 @@ are not interested in such anomalies. Ideally, we could instruct the detector to
 skip results for domains that we consider safe. Using a rule with a scope allows 
 us to achieve this.
 
-First, we need to create a list with our safe domains. Those lists are called 
-`filters` in {ml}. Filters can be shared across jobs.
+First, we need to create a list of our safe domains. Those lists are called 
+_filters_ in {ml}. Filters can be shared across jobs.
 
 We create our filter using the {ref}/ml-put-filter.html[put filter API]:
 
@@ -40,8 +40,8 @@ PUT _xpack/ml/filters/safe_domains
 ----------------------------------
 // CONSOLE
 
-Now, we can create our job specifying a scope that uses the filter for the 
-`highest_registered_domain` field:
+Now, we can create our job specifying a scope that uses the `safe_domains` 
+filter for the `highest_registered_domain` field:
 
 [source,js]
 ----------------------------------
@@ -85,7 +85,9 @@ POST _xpack/ml/filters/safe_domains/_update
 // CONSOLE
 // TEST[setup:ml_filter_safe_domains]
 
-Note that we can provide scope for any of the partition/over/by fields.
+Note that we can use any of the `partition_field_name`, `over_field_name`, or 
+`by_field_name` fields in the `scope`.
+
 In the following example we scope multiple fields:
 
 [source,js]
@@ -210,9 +212,9 @@ Rules only affect results created after the rules were applied.
 Let us imagine that we have configured a job and it has been running
 for some time. After observing its results we decide that we can employ
 rules in order to get rid of some uninteresting results. We can use
-the update-job API to do so. However, the rule we added will only be in effect
-for any results created from the moment we added the rule onwards. Past results
-will remain unaffected.
+the {ref}/ml-update-job.html[update job API] to do so. However, the rule we 
+added will only be in effect for any results created from the moment we added 
+the rule onwards. Past results will remain unaffected.
 
 ==== Using rules VS filtering data
 

From 4b199dde8d245a2e0b81e0b556eb22eab637cecf Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Wed, 1 Aug 2018 02:34:58 +0200
Subject: [PATCH 255/260] NETWORKING: Fix Netty Leaks by upgrading to 4.1.28
 (#32511)

* Upgrade to `4.1.28` since the problem reported in #32487 is a bug in Netty itself (see https://github.com/netty/netty/issues/7337)
* Fixed other leaks in test code that now showed up due to fixes improvements in leak reporting in the newer version
* Needed to extend permissions for netty common package because it now sets a classloader at runtime after changes in https://github.com/netty/netty/commit/63bae0956aeb42e875ad303a2281989ad92acbf2
* Adjusted forbidden APIs check accordingly
* Closes #32487
---
 modules/transport-netty4/build.gradle         | 17 ++++----
 .../netty-buffer-4.1.16.Final.jar.sha1        |  1 -
 .../netty-buffer-4.1.28.Final.jar.sha1        |  1 +
 .../netty-codec-4.1.16.Final.jar.sha1         |  1 -
 .../netty-codec-4.1.28.Final.jar.sha1         |  1 +
 .../netty-codec-http-4.1.16.Final.jar.sha1    |  1 -
 .../netty-codec-http-4.1.28.Final.jar.sha1    |  1 +
 .../netty-common-4.1.16.Final.jar.sha1        |  1 -
 .../netty-common-4.1.28.Final.jar.sha1        |  1 +
 .../netty-handler-4.1.16.Final.jar.sha1       |  1 -
 .../netty-handler-4.1.28.Final.jar.sha1       |  1 +
 .../netty-resolver-4.1.16.Final.jar.sha1      |  1 -
 .../netty-resolver-4.1.28.Final.jar.sha1      |  1 +
 .../netty-transport-4.1.16.Final.jar.sha1     |  1 -
 .../netty-transport-4.1.28.Final.jar.sha1     |  1 +
 .../plugin-metadata/plugin-security.policy    |  3 ++
 .../http/netty4/Netty4BadRequestTests.java    | 19 +++++----
 .../netty4/Netty4HttpRequestSizeLimitIT.java  | 27 +++++++++----
 .../Netty4HttpServerPipeliningTests.java      |  9 ++++-
 .../Netty4HttpServerTransportTests.java       | 33 +++++++++++-----
 .../http/netty4/Netty4PipeliningIT.java       | 13 +++++--
 plugins/transport-nio/build.gradle            | 17 ++++----
 .../netty-buffer-4.1.16.Final.jar.sha1        |  1 -
 .../netty-buffer-4.1.28.Final.jar.sha1        |  1 +
 .../netty-codec-4.1.16.Final.jar.sha1         |  1 -
 .../netty-codec-4.1.28.Final.jar.sha1         |  1 +
 .../netty-codec-http-4.1.16.Final.jar.sha1    |  1 -
 .../netty-codec-http-4.1.28.Final.jar.sha1    |  1 +
 .../netty-common-4.1.16.Final.jar.sha1        |  1 -
 .../netty-common-4.1.28.Final.jar.sha1        |  1 +
 .../netty-handler-4.1.16.Final.jar.sha1       |  1 -
 .../netty-handler-4.1.28.Final.jar.sha1       |  1 +
 .../netty-resolver-4.1.16.Final.jar.sha1      |  1 -
 .../netty-resolver-4.1.28.Final.jar.sha1      |  1 +
 .../netty-transport-4.1.16.Final.jar.sha1     |  1 -
 .../netty-transport-4.1.28.Final.jar.sha1     |  1 +
 .../plugin-metadata/plugin-security.policy    |  2 +
 .../http/nio/NioHttpServerTransportTests.java | 39 ++++++++++++-------
 .../plugin-metadata/plugin-security.policy    |  2 +
 .../plugin-metadata/plugin-security.policy    |  2 +
 40 files changed, 136 insertions(+), 75 deletions(-)
 delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.28.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.28.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.28.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-common-4.1.28.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.28.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.28.Final.jar.sha1
 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1
 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.28.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.28.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.28.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.28.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-common-4.1.28.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.28.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.28.Final.jar.sha1
 delete mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1
 create mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.28.Final.jar.sha1

diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index 5d4bcd7c10a..12ce5ce7d4a 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -34,13 +34,13 @@ compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-tr
 
 dependencies {
     // network stack
-    compile "io.netty:netty-buffer:4.1.16.Final"
-    compile "io.netty:netty-codec:4.1.16.Final"
-    compile "io.netty:netty-codec-http:4.1.16.Final"
-    compile "io.netty:netty-common:4.1.16.Final"
-    compile "io.netty:netty-handler:4.1.16.Final"
-    compile "io.netty:netty-resolver:4.1.16.Final"
-    compile "io.netty:netty-transport:4.1.16.Final"
+    compile "io.netty:netty-buffer:4.1.28.Final"
+    compile "io.netty:netty-codec:4.1.28.Final"
+    compile "io.netty:netty-codec-http:4.1.28.Final"
+    compile "io.netty:netty-common:4.1.28.Final"
+    compile "io.netty:netty-handler:4.1.28.Final"
+    compile "io.netty:netty-resolver:4.1.28.Final"
+    compile "io.netty:netty-transport:4.1.28.Final"
 }
 
 dependencyLicenses {
@@ -134,7 +134,6 @@ thirdPartyAudit.excludes = [
         'net.jpountz.xxhash.StreamingXXHash32',
         'net.jpountz.xxhash.XXHashFactory',
         'io.netty.internal.tcnative.CertificateRequestedCallback',
-        'io.netty.internal.tcnative.CertificateRequestedCallback$KeyMaterial',
         'io.netty.internal.tcnative.CertificateVerifier',
         'io.netty.internal.tcnative.SessionTicketKey',
         'io.netty.internal.tcnative.SniHostNameMatcher',
@@ -161,6 +160,6 @@ thirdPartyAudit.excludes = [
 
         'org.conscrypt.AllocatedBuffer',
         'org.conscrypt.BufferAllocator',
-        'org.conscrypt.Conscrypt$Engines',
+        'org.conscrypt.Conscrypt',
         'org.conscrypt.HandshakeListener'
 ]
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1
deleted file mode 100644
index c5462229719..00000000000
--- a/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-63b5fa95c74785e16f2c30ce268bc222e35c8cb5
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..f8a652d0dd1
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-buffer-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+d6c2d13492778009d33f60e05ed90bcb535d1fd1
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1
deleted file mode 100644
index 1e6c241ea0b..00000000000
--- a/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d84a1f21768b7309c2954521cf5a1f46c2309eb1
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..70799bf1032
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+a38361d893900947524f8a9da980555950e73d6a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1
deleted file mode 100644
index 71c33af1c5f..00000000000
--- a/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d64312378b438dfdad84267c599a053327c6f02a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..e1d34ebf89b
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+897100c1022c780b0a436b9349e507e8fa9800dc
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1
deleted file mode 100644
index 3edf5fcea59..00000000000
--- a/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-177a6b30cca92f6f5f9873c9befd681377a4c328
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..bc951426832
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-common-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+df69ce8bb9b544a71e7bbee290253cf7c93e6bad
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1
deleted file mode 100644
index cba27387268..00000000000
--- a/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fec0e63e7dd7f4eeef7ea8dc47a1ff32dfc7ebc2
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..80dc8b8f6fe
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-handler-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+a035784682da0126bc25f10713dac732b5082a6d
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1
deleted file mode 100644
index 3571d2ecfdc..00000000000
--- a/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f6eb553b53fb3a90a8ac1170697093fed82eae28
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..afe004bd716
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-resolver-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+f33557dcb31fa20da075ac05e4808115e32ef9b7
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1
deleted file mode 100644
index e502d4c7708..00000000000
--- a/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3c8ee2c4d4a1cbb947a5c184c7aeb2204260958b
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.28.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..af19a16d6ed
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+d2ef28f49d726737f0ffe84bf66529b3bf6e0c0d
\ No newline at end of file
diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
index 32b2dc9bd15..b8099025e47 100644
--- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
@@ -23,6 +23,9 @@ grant codeBase "${codebase.netty-common}" {
 
    // netty makes and accepts socket connections
    permission java.net.SocketPermission "*", "accept,connect";
+
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
index 094f3390598..a005ae10546 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -92,15 +93,19 @@ public class Netty4BadRequestTests extends ESTestCase {
             try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
                 final Collection<FullHttpResponse> responses =
                         nettyHttpClient.get(transportAddress.address(), "/_cluster/settings?pretty=%");
-                assertThat(responses, hasSize(1));
-                assertThat(responses.iterator().next().status().code(), equalTo(400));
-                final Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
-                assertThat(responseBodies, hasSize(1));
-                assertThat(responseBodies.iterator().next(), containsString("\"type\":\"bad_parameter_exception\""));
-                assertThat(
+                try {
+                    assertThat(responses, hasSize(1));
+                    assertThat(responses.iterator().next().status().code(), equalTo(400));
+                    final Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
+                    assertThat(responseBodies, hasSize(1));
+                    assertThat(responseBodies.iterator().next(), containsString("\"type\":\"bad_parameter_exception\""));
+                    assertThat(
                         responseBodies.iterator().next(),
                         containsString(
-                                "\"reason\":\"java.lang.IllegalArgumentException: unterminated escape sequence at end of string: %\""));
+                            "\"reason\":\"java.lang.IllegalArgumentException: unterminated escape sequence at end of string: %\""));
+                } finally {
+                    responses.forEach(ReferenceCounted::release);
+                }
             }
         }
     }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
index 269773fbb63..ad81f5b3063 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ESNetty4IntegTestCase;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
@@ -88,12 +89,20 @@ public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase {
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> singleResponse = nettyHttpClient.post(transportAddress.address(), requests[0]);
-            assertThat(singleResponse, hasSize(1));
-            assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK);
+            try {
+                assertThat(singleResponse, hasSize(1));
+                assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK);
 
-            Collection<FullHttpResponse> multipleResponses = nettyHttpClient.post(transportAddress.address(), requests);
-            assertThat(multipleResponses, hasSize(requests.length));
-            assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE);
+                Collection<FullHttpResponse> multipleResponses = nettyHttpClient.post(transportAddress.address(), requests);
+                try {
+                    assertThat(multipleResponses, hasSize(requests.length));
+                    assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE);
+                } finally {
+                    multipleResponses.forEach(ReferenceCounted::release);
+                }
+            } finally {
+                singleResponse.forEach(ReferenceCounted::release);
+            }
         }
     }
 
@@ -113,8 +122,12 @@ public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase {
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> responses = nettyHttpClient.put(transportAddress.address(), requestUris);
-            assertThat(responses, hasSize(requestUris.length));
-            assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
+            try {
+                assertThat(responses, hasSize(requestUris.length));
+                assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
         }
     }
 
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
index 50be7f7ce45..cb4d14038d2 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
@@ -29,6 +29,7 @@ import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -98,8 +99,12 @@ public class Netty4HttpServerPipeliningTests extends ESTestCase {
 
             try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
                 Collection<FullHttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{}));
-                Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
-                assertThat(responseBodies, contains(requests.toArray()));
+                try {
+                    Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
+                    assertThat(responseBodies, contains(requests.toArray()));
+                } finally {
+                    responses.forEach(ReferenceCounted::release);
+                }
             }
         }
     }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
index bcf28506143..1c3c71d710d 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -207,14 +207,23 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
                 HttpUtil.setContentLength(request, contentLength);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(expectedStatus));
-                if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
-                    final FullHttpRequest continuationRequest =
-                        new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
-                    final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
-
-                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
-                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                try {
+                    assertThat(response.status(), equalTo(expectedStatus));
+                    if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
+                        final FullHttpRequest continuationRequest =
+                            new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
+                        final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
+                        try {
+                            assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                            assertThat(
+                                new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done")
+                            );
+                        } finally {
+                            continuationResponse.release();
+                        }
+                    }
+                } finally {
+                    response.release();
                 }
             }
         }
@@ -280,10 +289,14 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
                 final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
-                assertThat(
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
+                    assertThat(
                         new String(response.content().array(), Charset.forName("UTF-8")),
                         containsString("you sent a bad request and you should feel bad"));
+                } finally {
+                    response.release();
+                }
             }
         }
 
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
index ebb91d9663e..275442cd50a 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ESNetty4IntegTestCase;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
@@ -45,14 +46,18 @@ public class Netty4PipeliningIT extends ESNetty4IntegTestCase {
 
         HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
         TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses();
-        TransportAddress transportAddress = (TransportAddress) randomFrom(boundAddresses);
+        TransportAddress transportAddress = randomFrom(boundAddresses);
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests);
-            assertThat(responses, hasSize(5));
+            try {
+                assertThat(responses, hasSize(5));
 
-            Collection<String> opaqueIds = Netty4HttpClient.returnOpaqueIds(responses);
-            assertOpaqueIdsInOrder(opaqueIds);
+                Collection<String> opaqueIds = Netty4HttpClient.returnOpaqueIds(responses);
+                assertOpaqueIdsInOrder(opaqueIds);
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
         }
     }
 
diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle
index 856a8552b8e..07605bfee29 100644
--- a/plugins/transport-nio/build.gradle
+++ b/plugins/transport-nio/build.gradle
@@ -29,13 +29,13 @@ dependencies {
     compile "org.elasticsearch:elasticsearch-nio:${version}"
 
     // network stack
-    compile "io.netty:netty-buffer:4.1.16.Final"
-    compile "io.netty:netty-codec:4.1.16.Final"
-    compile "io.netty:netty-codec-http:4.1.16.Final"
-    compile "io.netty:netty-common:4.1.16.Final"
-    compile "io.netty:netty-handler:4.1.16.Final"
-    compile "io.netty:netty-resolver:4.1.16.Final"
-    compile "io.netty:netty-transport:4.1.16.Final"
+    compile "io.netty:netty-buffer:4.1.28.Final"
+    compile "io.netty:netty-codec:4.1.28.Final"
+    compile "io.netty:netty-codec-http:4.1.28.Final"
+    compile "io.netty:netty-common:4.1.28.Final"
+    compile "io.netty:netty-handler:4.1.28.Final"
+    compile "io.netty:netty-resolver:4.1.28.Final"
+    compile "io.netty:netty-transport:4.1.28.Final"
 }
 
 dependencyLicenses {
@@ -113,7 +113,6 @@ thirdPartyAudit.excludes = [
         'net.jpountz.xxhash.StreamingXXHash32',
         'net.jpountz.xxhash.XXHashFactory',
         'io.netty.internal.tcnative.CertificateRequestedCallback',
-        'io.netty.internal.tcnative.CertificateRequestedCallback$KeyMaterial',
         'io.netty.internal.tcnative.CertificateVerifier',
         'io.netty.internal.tcnative.SessionTicketKey',
         'io.netty.internal.tcnative.SniHostNameMatcher',
@@ -140,6 +139,6 @@ thirdPartyAudit.excludes = [
 
         'org.conscrypt.AllocatedBuffer',
         'org.conscrypt.BufferAllocator',
-        'org.conscrypt.Conscrypt$Engines',
+        'org.conscrypt.Conscrypt',
         'org.conscrypt.HandshakeListener'
 ]
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1
deleted file mode 100644
index c5462229719..00000000000
--- a/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-63b5fa95c74785e16f2c30ce268bc222e35c8cb5
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..f8a652d0dd1
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-buffer-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+d6c2d13492778009d33f60e05ed90bcb535d1fd1
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1
deleted file mode 100644
index 1e6c241ea0b..00000000000
--- a/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d84a1f21768b7309c2954521cf5a1f46c2309eb1
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..70799bf1032
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+a38361d893900947524f8a9da980555950e73d6a
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1
deleted file mode 100644
index 71c33af1c5f..00000000000
--- a/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d64312378b438dfdad84267c599a053327c6f02a
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..e1d34ebf89b
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+897100c1022c780b0a436b9349e507e8fa9800dc
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1
deleted file mode 100644
index 3edf5fcea59..00000000000
--- a/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-177a6b30cca92f6f5f9873c9befd681377a4c328
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..bc951426832
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-common-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+df69ce8bb9b544a71e7bbee290253cf7c93e6bad
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1
deleted file mode 100644
index cba27387268..00000000000
--- a/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fec0e63e7dd7f4eeef7ea8dc47a1ff32dfc7ebc2
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..80dc8b8f6fe
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-handler-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+a035784682da0126bc25f10713dac732b5082a6d
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1
deleted file mode 100644
index 3571d2ecfdc..00000000000
--- a/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f6eb553b53fb3a90a8ac1170697093fed82eae28
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..afe004bd716
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-resolver-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+f33557dcb31fa20da075ac05e4808115e32ef9b7
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1
deleted file mode 100644
index e502d4c7708..00000000000
--- a/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3c8ee2c4d4a1cbb947a5c184c7aeb2204260958b
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.28.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.28.Final.jar.sha1
new file mode 100644
index 00000000000..af19a16d6ed
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-transport-4.1.28.Final.jar.sha1
@@ -0,0 +1 @@
+d2ef28f49d726737f0ffe84bf66529b3bf6e0c0d
\ No newline at end of file
diff --git a/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy b/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy
index 8c8fe7c3274..1c6eb4f10e3 100644
--- a/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy
@@ -26,4 +26,6 @@ grant codeBase "${codebase.netty-common}" {
    // This should only currently be required as we use the netty http client for tests
    // netty makes and accepts socket connections
    permission java.net.SocketPermission "*", "accept,connect";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
index a0cb74f7cd2..c7e92358531 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
@@ -198,14 +198,23 @@ public class NioHttpServerTransportTests extends ESTestCase {
                 HttpUtil.setContentLength(request, contentLength);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(expectedStatus));
-                if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
-                    final FullHttpRequest continuationRequest =
-                        new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
-                    final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
-
-                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
-                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                try {
+                    assertThat(response.status(), equalTo(expectedStatus));
+                    if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
+                        final FullHttpRequest continuationRequest =
+                            new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
+                        final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
+                        try {
+                            assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                            assertThat(
+                                new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done")
+                            );
+                        } finally {
+                            continuationResponse.release();
+                        }
+                    }
+                } finally {
+                    response.release();
                 }
             }
         }
@@ -271,17 +280,21 @@ public class NioHttpServerTransportTests extends ESTestCase {
                 final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
-                assertThat(
-                    new String(response.content().array(), Charset.forName("UTF-8")),
-                    containsString("you sent a bad request and you should feel bad"));
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
+                    assertThat(
+                        new String(response.content().array(), Charset.forName("UTF-8")),
+                        containsString("you sent a bad request and you should feel bad"));
+                } finally {
+                    response.release();
+                }
             }
         }
 
         assertNotNull(causeReference.get());
         assertThat(causeReference.get(), instanceOf(TooLongFrameException.class));
     }
-    
+
 //    public void testReadTimeout() throws Exception {
 //        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
 //
diff --git a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
index 0cd7a32bcc4..70abb67a762 100644
--- a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
@@ -15,6 +15,8 @@ grant {
 grant codeBase "${codebase.netty-common}" {
    // for reading the system-wide configuration for the backlog of established sockets
    permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {
diff --git a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
index 9ba8ea6798b..79fced09adb 100644
--- a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
@@ -9,6 +9,8 @@ grant {
 grant codeBase "${codebase.netty-common}" {
    // for reading the system-wide configuration for the backlog of established sockets
    permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {

From 99d9a0a40c53b628bcff4353ae7e80ccb934769b Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 31 Jul 2018 21:04:33 -0400
Subject: [PATCH 256/260] Switch x-pack rolling restart to new style Requests
 (#32339)

In #29623 we added `Request` object flavored requests to the low level
REST client and in #30315 we deprecated the old `performRequest`s. This
changes all calls in the `x-pack:qa:rolling-upgrade*` projects to use
the new versions.
---
 .../upgrades/BasicLicenseUpgradeIT.java       |   5 +-
 .../upgrades/AbstractUpgradeTestCase.java     |   6 +-
 .../upgrades/IndexAuditUpgradeIT.java         |  20 ++-
 .../TokenBackwardsCompatibilityIT.java        | 120 ++++++++++--------
 4 files changed, 81 insertions(+), 70 deletions(-)

diff --git a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java
index 2967096b00b..924479b4f88 100644
--- a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.upgrades;
 
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 
 import java.util.Map;
@@ -22,7 +23,7 @@ public class BasicLicenseUpgradeIT extends AbstractUpgradeTestCase {
     }
 
     private void checkBasicLicense() throws Exception {
-        Response licenseResponse = client().performRequest("GET", "/_xpack/license");
+        Response licenseResponse = client().performRequest(new Request("GET", "/_xpack/license"));
         Map<String, Object> licenseResponseMap = entityAsMap(licenseResponse);
         Map<String, Object> licenseMap = (Map<String, Object>) licenseResponseMap.get("license");
         assertEquals("basic", licenseMap.get("type"));
@@ -30,7 +31,7 @@ public class BasicLicenseUpgradeIT extends AbstractUpgradeTestCase {
     }
 
     private void checkNonExpiringBasicLicense() throws Exception {
-        Response licenseResponse = client().performRequest("GET", "/_xpack/license");
+        Response licenseResponse = client().performRequest(new Request("GET", "/_xpack/license"));
         Map<String, Object> licenseResponseMap = entityAsMap(licenseResponse);
         Map<String, Object> licenseMap = (Map<String, Object>) licenseResponseMap.get("license");
         assertEquals("basic", licenseMap.get("type"));
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
index a3576b7b8c3..ec912a68b59 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.upgrades;
 
+import org.elasticsearch.client.Request;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.SecuritySettingsSourceField;
@@ -75,8 +76,9 @@ public abstract class AbstractUpgradeTestCase extends ESRestTestCase {
             boolean success = true;
             for (String template : templatesToWaitFor()) {
                 try {
-                    final boolean exists =
-                            adminClient().performRequest("HEAD", "_template/" + template).getStatusLine().getStatusCode() == 200;
+                    final boolean exists = adminClient()
+                            .performRequest(new Request("HEAD", "_template/" + template))
+                            .getStatusLine().getStatusCode() == 200;
                     success &= exists;
                     logger.debug("template [{}] exists [{}]", template, exists);
                 } catch (IOException e) {
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java
index 92a8c3d2f83..2a681afc14c 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java
@@ -5,16 +5,13 @@
  */
 package org.elasticsearch.upgrades;
 
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Booleans;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -52,7 +49,7 @@ public class IndexAuditUpgradeIT extends AbstractUpgradeTestCase {
     }
 
     private void assertAuditDocsExist() throws Exception {
-        Response response = client().performRequest("GET", "/.security_audit_log*/doc/_count");
+        Response response = client().performRequest(new Request("GET", "/.security_audit_log*/doc/_count"));
         assertEquals(200, response.getStatusLine().getStatusCode());
         Map<String, Object> responseMap = entityAsMap(response);
         assertNotNull(responseMap.get("count"));
@@ -61,19 +58,20 @@ public class IndexAuditUpgradeIT extends AbstractUpgradeTestCase {
 
     private void assertNumUniqueNodeNameBuckets(int numBuckets) throws Exception {
         // call API that will hit all nodes
-        Map<?, ?> nodesResponse = entityAsMap(client().performRequest("GET", "/_nodes/_all/info/version"));
+        Map<?, ?> nodesResponse = entityAsMap(client().performRequest(new Request("GET", "/_nodes/_all/info/version")));
         logger.info("all nodes {}", nodesResponse);
 
-        HttpEntity httpEntity = new StringEntity(
+        Request aggRequest = new Request("GET", "/.security_audit_log*/_search");
+        aggRequest.setJsonEntity(
                 "{\n" +
                 "    \"aggs\" : {\n" +
                 "        \"nodes\" : {\n" +
                 "            \"terms\" : { \"field\" : \"node_name\" }\n" +
                 "        }\n" +
                 "    }\n" +
-                "}", ContentType.APPLICATION_JSON);
-        Response aggResponse = client().performRequest("GET", "/.security_audit_log*/_search",
-                Collections.singletonMap("pretty", "true"), httpEntity);
+                "}");
+        aggRequest.addParameter("pretty", "true");
+        Response aggResponse = client().performRequest(aggRequest);
         Map<String, Object> aggResponseMap = entityAsMap(aggResponse);
         logger.debug("aggResponse {}", aggResponseMap);
         Map<?, ?> aggregations = (Map<?, ?>) aggResponseMap.get("aggregations");
@@ -89,7 +87,7 @@ public class IndexAuditUpgradeIT extends AbstractUpgradeTestCase {
      * Has the master been upgraded to the new version?
      */
     private boolean masterIsNewVersion() throws IOException {
-        Map<?, ?> map = entityAsMap(client().performRequest("GET", "/_nodes/_master"));
+        Map<?, ?> map = entityAsMap(client().performRequest(new Request("GET", "/_nodes/_master")));
         map = (Map<?, ?>) map.get("nodes");
         assertThat(map.values(), hasSize(1));
         map = (Map<?, ?>) map.values().iterator().next();
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
index 705122252e7..24965efc621 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
@@ -7,10 +7,9 @@ package org.elasticsearch.upgrades;
 
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpHost;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -18,7 +17,6 @@ import org.elasticsearch.test.rest.yaml.ObjectPath;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -26,43 +24,46 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
 
     public void testGeneratingTokenInOldCluster() throws Exception {
         assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD);
-        final StringEntity tokenPostBody = new StringEntity("{\n" +
+        Request createTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        createTokenRequest.setJsonEntity(
+                "{\n" +
                 "    \"username\": \"test_user\",\n" +
                 "    \"password\": \"x-pack-test-password\",\n" +
                 "    \"grant_type\": \"password\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        Response response = client().performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenPostBody);
+                "}");
+        Response response = client().performRequest(createTokenRequest);
         assertOK(response);
         Map<String, Object> responseMap = entityAsMap(response);
         String token = (String) responseMap.get("access_token");
         assertNotNull(token);
         assertTokenWorks(token);
 
-        StringEntity oldClusterToken = new StringEntity("{\n" +
+        Request indexRequest1 = new Request("PUT", "token_backwards_compatibility_it/doc/old_cluster_token1");
+        indexRequest1.setJsonEntity(
+                "{\n" +
                 "    \"token\": \"" + token + "\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        Response indexResponse = client().performRequest("PUT", "token_backwards_compatibility_it/doc/old_cluster_token1",
-                Collections.emptyMap(), oldClusterToken);
-        assertOK(indexResponse);
+                "}");
+        client().performRequest(indexRequest1);
 
-        response = client().performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenPostBody);
-        assertOK(response);
+        Request createSecondTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        createSecondTokenRequest.setEntity(createTokenRequest.getEntity());
+        response = client().performRequest(createSecondTokenRequest);
         responseMap = entityAsMap(response);
         token = (String) responseMap.get("access_token");
         assertNotNull(token);
         assertTokenWorks(token);
-        oldClusterToken = new StringEntity("{\n" +
+        Request indexRequest2 = new Request("PUT", "token_backwards_compatibility_it/doc/old_cluster_token2");
+        indexRequest2.setJsonEntity(
+                "{\n" +
                 "    \"token\": \"" + token + "\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        indexResponse = client().performRequest("PUT", "token_backwards_compatibility_it/doc/old_cluster_token2",
-                Collections.emptyMap(), oldClusterToken);
-        assertOK(indexResponse);
+                "}");
+        client().performRequest(indexRequest2);
     }
 
     public void testTokenWorksInMixedOrUpgradedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed or upgraded cluster",
                 CLUSTER_TYPE == ClusterType.MIXED || CLUSTER_TYPE == ClusterType.UPGRADED);
-        Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token1");
+        Response getResponse = client().performRequest(new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token1"));
         assertOK(getResponse);
         Map<String, Object> source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
         assertTokenWorks((String) source.get("token"));
@@ -71,26 +72,27 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
     public void testMixedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED);
         assumeTrue("the master must be on the latest version before we can write", isMasterOnLatestVersion());
-        Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token2");
-        assertOK(getResponse);
+        Response getResponse = client().performRequest(new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token2"));
         Map<String, Object> source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
         final String token = (String) source.get("token");
         assertTokenWorks(token);
 
-        final StringEntity body = new StringEntity("{\"token\": \"" + token + "\"}", ContentType.APPLICATION_JSON);
-        Response invalidationResponse = client().performRequest("DELETE", "_xpack/security/oauth2/token", Collections.emptyMap(), body);
-        assertOK(invalidationResponse);
+        Request invalidateRequest = new Request("DELETE", "_xpack/security/oauth2/token");
+        invalidateRequest.setJsonEntity("{\"token\": \"" + token + "\"}");
+        invalidateRequest.addParameter("error_trace", "true");
+        client().performRequest(invalidateRequest);
         assertTokenDoesNotWork(token);
 
         // create token and refresh on version that supports it
-        final StringEntity tokenPostBody = new StringEntity("{\n" +
+        Request createTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        createTokenRequest.setJsonEntity(
+                "{\n" +
                 "    \"username\": \"test_user\",\n" +
                 "    \"password\": \"x-pack-test-password\",\n" +
                 "    \"grant_type\": \"password\"\n" +
-                "}", ContentType.APPLICATION_JSON);
+                "}");
         try (RestClient client = getRestClientForCurrentVersionNodesOnly()) {
-            Response response = client.performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenPostBody);
-            assertOK(response);
+            Response response = client.performRequest(createTokenRequest);
             Map<String, Object> responseMap = entityAsMap(response);
             String accessToken = (String) responseMap.get("access_token");
             String refreshToken = (String) responseMap.get("refresh_token");
@@ -98,12 +100,13 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
             assertNotNull(refreshToken);
             assertTokenWorks(accessToken);
 
-            final StringEntity tokenRefresh = new StringEntity("{\n" +
+            Request tokenRefreshRequest = new Request("POST", "_xpack/security/oauth2/token");
+            tokenRefreshRequest.setJsonEntity(
+                    "{\n" +
                     "    \"refresh_token\": \"" + refreshToken + "\",\n" +
                     "    \"grant_type\": \"refresh_token\"\n" +
-                    "}", ContentType.APPLICATION_JSON);
-            response = client.performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenRefresh);
-            assertOK(response);
+                    "}");
+            response = client.performRequest(tokenRefreshRequest);
             responseMap = entityAsMap(response);
             String updatedAccessToken = (String) responseMap.get("access_token");
             String updatedRefreshToken = (String) responseMap.get("refresh_token");
@@ -118,31 +121,32 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
 
     public void testUpgradedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.UPGRADED);
-        Response getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token2");
+        Response getResponse = client().performRequest(new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token2"));
         assertOK(getResponse);
         Map<String, Object> source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
         final String token = (String) source.get("token");
 
         // invalidate again since this may not have been invalidated in the mixed cluster
-        final StringEntity body = new StringEntity("{\"token\": \"" + token + "\"}", ContentType.APPLICATION_JSON);
-        Response invalidationResponse = client().performRequest("DELETE", "_xpack/security/oauth2/token",
-                Collections.singletonMap("error_trace", "true"), body);
+        Request invalidateRequest = new Request("DELETE", "_xpack/security/oauth2/token");
+        invalidateRequest.setJsonEntity("{\"token\": \"" + token + "\"}");
+        invalidateRequest.addParameter("error_trace", "true");
+        Response invalidationResponse = client().performRequest(invalidateRequest);
         assertOK(invalidationResponse);
         assertTokenDoesNotWork(token);
 
-        getResponse = client().performRequest("GET", "token_backwards_compatibility_it/doc/old_cluster_token1");
-        assertOK(getResponse);
+        getResponse = client().performRequest(new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token1"));
         source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
         final String workingToken = (String) source.get("token");
         assertTokenWorks(workingToken);
 
-        final StringEntity tokenPostBody = new StringEntity("{\n" +
+        Request getTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        getTokenRequest.setJsonEntity(
+                "{\n" +
                 "    \"username\": \"test_user\",\n" +
                 "    \"password\": \"x-pack-test-password\",\n" +
                 "    \"grant_type\": \"password\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        Response response = client().performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenPostBody);
-        assertOK(response);
+                "}");
+        Response response = client().performRequest(getTokenRequest);
         Map<String, Object> responseMap = entityAsMap(response);
         String accessToken = (String) responseMap.get("access_token");
         String refreshToken = (String) responseMap.get("refresh_token");
@@ -150,12 +154,13 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
         assertNotNull(refreshToken);
         assertTokenWorks(accessToken);
 
-        final StringEntity tokenRefresh = new StringEntity("{\n" +
+        Request refreshTokenRequest = new Request("POST", "_xpack/security/oauth2/token");
+        refreshTokenRequest.setJsonEntity(
+                "{\n" +
                 "    \"refresh_token\": \"" + refreshToken + "\",\n" +
                 "    \"grant_type\": \"refresh_token\"\n" +
-                "}", ContentType.APPLICATION_JSON);
-        response = client().performRequest("POST", "_xpack/security/oauth2/token", Collections.emptyMap(), tokenRefresh);
-        assertOK(response);
+                "}");
+        response = client().performRequest(refreshTokenRequest);
         responseMap = entityAsMap(response);
         String updatedAccessToken = (String) responseMap.get("access_token");
         String updatedRefreshToken = (String) responseMap.get("refresh_token");
@@ -168,16 +173,21 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
     }
 
     private void assertTokenWorks(String token) throws IOException {
-        Response authenticateResponse = client().performRequest("GET", "_xpack/security/_authenticate", Collections.emptyMap(),
-                new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token));
+        Request request = new Request("GET", "_xpack/security/_authenticate");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token);
+        request.setOptions(options);
+        Response authenticateResponse = client().performRequest(request);
         assertOK(authenticateResponse);
         assertEquals("test_user", entityAsMap(authenticateResponse).get("username"));
     }
 
     private void assertTokenDoesNotWork(String token) {
-        ResponseException e = expectThrows(ResponseException.class,
-                () -> client().performRequest("GET", "_xpack/security/_authenticate", Collections.emptyMap(),
-                        new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token)));
+        Request request = new Request("GET", "_xpack/security/_authenticate");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + token);
+        request.setOptions(options);
+        ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
         assertEquals(401, e.getResponse().getStatusLine().getStatusCode());
         Response response = e.getResponse();
         assertEquals("Bearer realm=\"security\", error=\"invalid_token\", error_description=\"The access token expired\"",
@@ -185,17 +195,17 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
     }
 
     private boolean isMasterOnLatestVersion() throws Exception {
-        Response response = client().performRequest("GET", "_cluster/state");
+        Response response = client().performRequest(new Request("GET", "_cluster/state"));
         assertOK(response);
         final String masterNodeId = ObjectPath.createFromResponse(response).evaluate("master_node");
-        response = client().performRequest("GET", "_nodes");
+        response = client().performRequest(new Request("GET", "_nodes"));
         assertOK(response);
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
         return Version.CURRENT.equals(Version.fromString(objectPath.evaluate("nodes." + masterNodeId + ".version")));
     }
 
     private RestClient getRestClientForCurrentVersionNodesOnly() throws IOException {
-        Response response = client().performRequest("GET", "_nodes");
+        Response response = client().performRequest(new Request("GET", "_nodes"));
         assertOK(response);
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
         Map<String, Object> nodesAsMap = objectPath.evaluate("nodes");

From f2f33f31497654dc0fd687dd480e1636c898b566 Mon Sep 17 00:00:00 2001
From: Jay Modi <jaymode@users.noreply.github.com>
Date: Tue, 31 Jul 2018 20:57:33 -0600
Subject: [PATCH 257/260] Use hostname instead of IP with SPNEGO test (#32514)

This change updates KerberosAuthenticationIT to resolve the host used
to connect to the test cluster. This is needed because the host could
be an IP address but SPNEGO requires a hostname to work properly. This
is done by adding a hook in ESRestTestCase for building the HttpHost
from the host and port.

Additionally, the project now specifies the IPv4 loopback address as
the http host. This is done because we need to be able to resolve the
address used for the HTTP transport before the node starts up, but the
http.ports file is not written until the node is started.

Closes #32498
---
 .../test/rest/ESRestTestCase.java             |  9 ++-
 x-pack/qa/kerberos-tests/build.gradle         | 59 ++++++++++++-------
 .../kerberos/KerberosAuthenticationIT.java    | 17 +++++-
 3 files changed, 60 insertions(+), 25 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index e25e8b43d3a..ecb965040f8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -148,7 +148,7 @@ public abstract class ESRestTestCase extends ESTestCase {
                 }
                 String host = stringUrl.substring(0, portSeparator);
                 int port = Integer.valueOf(stringUrl.substring(portSeparator + 1));
-                hosts.add(new HttpHost(host, port, getProtocol()));
+                hosts.add(buildHttpHost(host, port));
             }
             clusterHosts = unmodifiableList(hosts);
             logger.info("initializing REST clients against {}", clusterHosts);
@@ -160,6 +160,13 @@ public abstract class ESRestTestCase extends ESTestCase {
         assert clusterHosts != null;
     }
 
+    /**
+     * Construct a HttpHost from the given host and port
+     */
+    protected HttpHost buildHttpHost(String host, int port) {
+        return new HttpHost(host, port, getProtocol());
+    }
+
     /**
      * Clean up after the test case.
      */
diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle
index d2818bfc127..7138b930512 100644
--- a/x-pack/qa/kerberos-tests/build.gradle
+++ b/x-pack/qa/kerberos-tests/build.gradle
@@ -36,32 +36,43 @@ task krb5kdcFixture(type: org.elasticsearch.gradle.test.VagrantFixture) {
     dependsOn krb5kdcUpdate
 }
 
-task krb5AddPrincipals { dependsOn krb5kdcFixture }
+// lazily resolve to avoid any slowdowns from DNS lookups prior to when we need this value
+Object httpPrincipal = new Object() {
+    @Override
+    String toString() {
+        InetAddress resolvedAddress = InetAddress.getByName('127.0.0.1')
+        return "HTTP/" + resolvedAddress.getHostName()
+    }
+}
 
-List<String> principals = [
-    "HTTP/localhost",
-    "peppa",
-    "george~dino"
-]
 String realm = "BUILD.ELASTIC.CO"
 
-for (String principal : principals) {
-    String[] princPwdPair = principal.split('~');
-    String princName = princPwdPair[0];
-    String password = "";
-    if (princPwdPair.length > 1) {
-        password = princPwdPair[1];
-    }
-    Task create = project.tasks.create("addPrincipal#${principal}".replace('/', '_'), org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
-        command 'ssh'
-        args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $princName $password"
-        boxName box
-        environmentVars vagrantEnvVars
-        dependsOn krb5kdcFixture
-    }
-    krb5AddPrincipals.dependsOn(create)
+task 'addPrincipal#peppa'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh peppa "
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
 }
 
+task 'addPrincipal#george'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh george dino"
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
+}
+
+task 'addPrincipal#HTTP'(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
+    command 'ssh'
+    args '--command', "sudo bash /vagrant/src/main/resources/provision/addprinc.sh $httpPrincipal"
+    boxName box
+    environmentVars vagrantEnvVars
+    dependsOn krb5kdcFixture
+}
+
+task krb5AddPrincipals { dependsOn krb5kdcFixture, 'addPrincipal#peppa', 'addPrincipal#george', 'addPrincipal#HTTP' }
+
 def generatedResources = "$buildDir/generated-resources/keytabs"
 task copyKeytabToGeneratedResources(type: Copy) {
     Path peppaKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("peppa.keytab").toAbsolutePath()
@@ -71,6 +82,9 @@ task copyKeytabToGeneratedResources(type: Copy) {
 }
 
 integTestCluster {
+    // force localhost IPv4 otherwise it is a chicken and egg problem where we need the keytab for the hostname when starting the cluster
+    // but do not know the exact address that is first in the http ports file
+    setting 'http.host', '127.0.0.1'
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.security.authc.realms.file.type', 'file'
@@ -87,7 +101,8 @@ integTestCluster {
     Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
     String jvmArgsStr = " -Djava.security.krb5.conf=${krb5conf}" + " -Dsun.security.krb5.debug=true"
     jvmArgs jvmArgsStr
-    Path esKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("HTTP_localhost.keytab").toAbsolutePath()
+    Path esKeytab = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs")
+            .resolve("$httpPrincipal".replace('/', '_') + ".keytab").toAbsolutePath()
     extraConfigFile("es.keytab", "${esKeytab}")
 
     setupCommand 'setupTestAdmin',
diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
index a36040d1ba8..ed9f4fbe38d 100644
--- a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
+++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
@@ -13,6 +13,7 @@ import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -23,6 +24,8 @@ import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
@@ -82,7 +85,6 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         assertOK(response);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32498")
     public void testLoginByKeytab() throws IOException, PrivilegedActionException {
         final String userPrincipalName = System.getProperty(TEST_USER_WITH_KEYTAB_KEY);
         final String keytabPath = System.getProperty(TEST_USER_WITH_KEYTAB_PATH_KEY);
@@ -92,7 +94,6 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         executeRequestAndVerifyResponse(userPrincipalName, callbackHandler);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32498")
     public void testLoginByUsernamePassword() throws IOException, PrivilegedActionException {
         final String userPrincipalName = System.getProperty(TEST_USER_WITH_PWD_KEY);
         final String password = System.getProperty(TEST_USER_WITH_PWD_PASSWD_KEY);
@@ -106,6 +107,18 @@ public class KerberosAuthenticationIT extends ESRestTestCase {
         // intentionally empty - this is just needed to ensure the build does not fail because we mute its only test.
     }
 
+    @Override
+    @SuppressForbidden(reason = "SPNEGO relies on hostnames and we need to ensure host isn't a IP address")
+    protected HttpHost buildHttpHost(String host, int port) {
+        try {
+            InetAddress inetAddress = InetAddress.getByName(host);
+            return super.buildHttpHost(inetAddress.getHostName(), port);
+        } catch (UnknownHostException e) {
+            assumeNoException("failed to resolve host [" + host + "]", e);
+        }
+        throw new IllegalStateException("DNS not resolved and assume did not trip");
+    }
+
     private void executeRequestAndVerifyResponse(final String userPrincipalName,
             final SpnegoHttpClientConfigCallbackHandler callbackHandler) throws PrivilegedActionException, IOException {
         final Request request = new Request("GET", "/_xpack/security/_authenticate");

From 82fe67b2250567dbe5a7cc8c9021af0c5fa4da38 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Wed, 1 Aug 2018 09:43:32 +0200
Subject: [PATCH 258/260] [Rollup] Remove builders from TermsGroupConfig
 (#32507)

While working on adding the Create Rollup Job API to the
high level REST client (#29827), I noticed that the configuration
objects like TermsGroupConfig rely on the Builder pattern in
order to create or parse instances. These builders are doing
some validation but the same validation could be done within
the constructor itself or on the server side when appropriate.

This commit removes the builder for TermsGroupConfig,
removes some other methods that I consider not really usefull
once the TermsGroupConfig object will be exposed in the
high level REST client. It also simplifies the parsing logic.

Related to #29827
---
 .../xpack/core/rollup/job/GroupConfig.java    | 13 ++-
 .../core/rollup/job/TermsGroupConfig.java     | 83 ++++++++-----------
 .../xpack/core/rollup/ConfigTestHelpers.java  | 29 +++++--
 .../job/TermsGroupConfigSerializingTests.java | 44 +++-------
 .../xpack/rollup/job/RollupIndexer.java       |  1 -
 .../rollup/RollupJobIdentifierUtilTests.java  |  5 +-
 .../rollup/action/SearchActionTests.java      |  7 +-
 .../xpack/rollup/config/ConfigTests.java      | 15 ++--
 .../xpack/rollup/job/IndexerUtilsTests.java   |  2 +-
 9 files changed, 88 insertions(+), 111 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java
index dcb393b0886..f6977394f4a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/GroupConfig.java
@@ -23,6 +23,8 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
+import static java.util.Arrays.asList;
+
 /**
  * The configuration object for the groups section in the rollup config.
  * Basically just a wrapper for histo/date histo/terms objects
@@ -50,7 +52,7 @@ public class GroupConfig implements Writeable, ToXContentObject {
     static {
         PARSER.declareObject(GroupConfig.Builder::setDateHisto, (p,c) -> DateHistoGroupConfig.PARSER.apply(p,c).build(), DATE_HISTO);
         PARSER.declareObject(GroupConfig.Builder::setHisto, (p,c) -> HistoGroupConfig.PARSER.apply(p,c).build(), HISTO);
-        PARSER.declareObject(GroupConfig.Builder::setTerms, (p,c) -> TermsGroupConfig.PARSER.apply(p,c).build(), TERMS);
+        PARSER.declareObject(GroupConfig.Builder::setTerms, (p,c) -> TermsGroupConfig.fromXContent(p), TERMS);
     }
 
     private GroupConfig(DateHistoGroupConfig dateHisto, @Nullable HistoGroupConfig histo, @Nullable TermsGroupConfig terms) {
@@ -84,7 +86,7 @@ public class GroupConfig implements Writeable, ToXContentObject {
             fields.addAll(histo.getAllFields());
         }
         if (terms != null) {
-            fields.addAll(terms.getAllFields());
+            fields.addAll(asList(terms.getFields()));
         }
         return fields;
     }
@@ -100,7 +102,6 @@ public class GroupConfig implements Writeable, ToXContentObject {
         }
     }
 
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -113,9 +114,7 @@ public class GroupConfig implements Writeable, ToXContentObject {
             builder.endObject();
         }
         if (terms != null) {
-            builder.startObject(TERMS.getPreferredName());
-            terms.toXContent(builder, params);
-            builder.endObject();
+            builder.field(TERMS.getPreferredName(), terms);
         }
         builder.endObject();
         return builder;
@@ -194,4 +193,4 @@ public class GroupConfig implements Writeable, ToXContentObject {
             return new GroupConfig(dateHisto, histo, terms);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
index da73020f008..a1b0b3118ec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfig.java
@@ -12,9 +12,10 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
@@ -24,13 +25,13 @@ import org.elasticsearch.xpack.core.rollup.RollupField;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * The configuration object for the histograms in the rollup config
  *
@@ -42,20 +43,29 @@ import java.util.stream.Collectors;
  *     ]
  * }
  */
-public class TermsGroupConfig implements Writeable, ToXContentFragment {
-    private static final String NAME = "term_group_config";
-    public static final ObjectParser<TermsGroupConfig.Builder, Void> PARSER = new ObjectParser<>(NAME, TermsGroupConfig.Builder::new);
+public class TermsGroupConfig implements Writeable, ToXContentObject {
+
+    private static final String NAME = "terms";
+    private static final String FIELDS = "fields";
 
-    private static final ParseField FIELDS = new ParseField("fields");
     private static final List<String> FLOAT_TYPES = Arrays.asList("half_float", "float", "double", "scaled_float");
     private static final List<String> NATURAL_TYPES = Arrays.asList("byte", "short", "integer", "long");
-    private final String[] fields;
 
+    private static final ConstructingObjectParser<TermsGroupConfig, Void> PARSER;
     static {
-        PARSER.declareStringArray(TermsGroupConfig.Builder::setFields, FIELDS);
+        PARSER = new ConstructingObjectParser<>(NAME, args -> {
+            @SuppressWarnings("unchecked") List<String> fields = (List<String>) args[0];
+            return new TermsGroupConfig(fields != null ? fields.toArray(new String[fields.size()]) : null);
+        });
+        PARSER.declareStringArray(constructorArg(), new ParseField(FIELDS));
     }
 
-    private TermsGroupConfig(String[] fields) {
+    private final String[] fields;
+
+    public TermsGroupConfig(final String... fields) {
+        if (fields == null || fields.length == 0) {
+            throw new IllegalArgumentException("Fields must have at least one value");
+        }
         this.fields = fields;
     }
 
@@ -63,6 +73,9 @@ public class TermsGroupConfig implements Writeable, ToXContentFragment {
         fields = in.readStringArray();
     }
 
+    /**
+     * @return the names of the fields. Never {@code null}.
+     */
     public String[] getFields() {
         return fields;
     }
@@ -72,10 +85,6 @@ public class TermsGroupConfig implements Writeable, ToXContentFragment {
      * set of date histograms.  Used by the rollup indexer to iterate over historical data
      */
     public List<CompositeValuesSourceBuilder<?>> toBuilders() {
-        if (fields.length == 0) {
-            return Collections.emptyList();
-        }
-
         return Arrays.stream(fields).map(f -> {
             TermsValuesSourceBuilder vsBuilder
                     = new TermsValuesSourceBuilder(RollupField.formatIndexerAggName(f, TermsAggregationBuilder.NAME));
@@ -94,14 +103,6 @@ public class TermsGroupConfig implements Writeable, ToXContentFragment {
         return map;
     }
 
-    public Map<String, Object> getMetadata() {
-        return Collections.emptyMap();
-    }
-
-    public Set<String> getAllFields() {
-        return Arrays.stream(fields).collect(Collectors.toSet());
-    }
-
     public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
                                  ActionRequestValidationException validationException) {
 
@@ -138,8 +139,11 @@ public class TermsGroupConfig implements Writeable, ToXContentFragment {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(FIELDS.getPreferredName(), fields);
-        return builder;
+        builder.startObject();
+        {
+            builder.field(FIELDS, fields);
+        }
+        return builder.endObject();
     }
 
     @Override
@@ -148,18 +152,15 @@ public class TermsGroupConfig implements Writeable, ToXContentFragment {
     }
 
     @Override
-    public boolean equals(Object other) {
+    public boolean equals(final Object other) {
         if (this == other) {
             return true;
         }
-
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
-
-        TermsGroupConfig that = (TermsGroupConfig) other;
-
-        return Arrays.equals(this.fields, that.fields);
+        final TermsGroupConfig that = (TermsGroupConfig) other;
+        return Arrays.equals(fields, that.fields);
     }
 
     @Override
@@ -172,23 +173,7 @@ public class TermsGroupConfig implements Writeable, ToXContentFragment {
         return Strings.toString(this, true, true);
     }
 
-    public static class Builder {
-        private List<String> fields;
-
-        public List<String> getFields() {
-            return fields;
-        }
-
-        public TermsGroupConfig.Builder setFields(List<String> fields) {
-            this.fields = fields;
-            return this;
-        }
-
-        public TermsGroupConfig build() {
-            if (fields == null || fields.isEmpty()) {
-                throw new IllegalArgumentException("Parameter [" + FIELDS + "] must have at least one value.");
-            }
-            return new TermsGroupConfig(fields.toArray(new String[0]));
-        }
+    public static TermsGroupConfig fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
index 3e4e4a84d2f..b2e2730760f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java
@@ -17,9 +17,13 @@ import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween;
+
 public class ConfigTestHelpers {
 
     public static RollupJobConfig.Builder getRollupJob(String jobId) {
@@ -49,7 +53,7 @@ public class ConfigTestHelpers {
             groupBuilder.setHisto(getHisto().build());
         }
         if (ESTestCase.randomBoolean()) {
-            groupBuilder.setTerms(getTerms().build());
+            groupBuilder.setTerms(randomTermsGroupConfig(ESTestCase.random()));
         }
         return groupBuilder;
     }
@@ -105,12 +109,6 @@ public class ConfigTestHelpers {
         return histoBuilder;
     }
 
-    public static TermsGroupConfig.Builder getTerms() {
-        TermsGroupConfig.Builder builder = new TermsGroupConfig.Builder();
-        builder.setFields(getFields());
-        return builder;
-    }
-
     public static  List<String> getFields() {
         return IntStream.range(0, ESTestCase.randomIntBetween(1, 10))
                 .mapToObj(n -> ESTestCase.randomAlphaOfLengthBetween(5, 10))
@@ -126,4 +124,21 @@ public class ConfigTestHelpers {
                 " ?"                                                                         + //day of week
                 " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(1970, 2199)));  //year
     }
+
+    public static TermsGroupConfig randomTermsGroupConfig(final Random random) {
+        return new TermsGroupConfig(randomFields(random));
+    }
+
+    private static String[] randomFields(final Random random) {
+        final int numFields = randomIntBetween(random, 1, 10);
+        final String[] fields = new String[numFields];
+        for (int i = 0; i < numFields; i++) {
+            fields[i] = randomField(random);
+        }
+        return fields;
+    }
+
+    private static String randomField(final Random random) {
+        return randomAsciiAlphanumOfLengthBetween(random, 5, 10);
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
index f48f678fac8..ccdd616df7b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
@@ -11,27 +11,23 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomTermsGroupConfig;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class TermsGroupConfigSerializingTests extends AbstractSerializingTestCase<TermsGroupConfig> {
 
-    private static final List<String> FLOAT_TYPES = Arrays.asList("half_float", "float", "double", "scaled_float");
-    private static final List<String> NATURAL_TYPES = Arrays.asList("byte", "short", "integer", "long");
-
     @Override
     protected TermsGroupConfig doParseInstance(XContentParser parser) throws IOException {
-        return TermsGroupConfig.PARSER.apply(parser, null).build();
+        return TermsGroupConfig.fromXContent(parser);
     }
 
     @Override
@@ -41,23 +37,20 @@ public class TermsGroupConfigSerializingTests extends AbstractSerializingTestCas
 
     @Override
     protected TermsGroupConfig createTestInstance() {
-        return ConfigTestHelpers.getTerms().build();
+        return randomTermsGroupConfig(random());
     }
 
-    public void testValidateNoMapping() throws IOException {
+    public void testValidateNoMapping() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [keyword/text] field with name " +
                 "[my_field] in any of the indices matching the index pattern."));
     }
 
-    public void testValidateNomatchingField() throws IOException {
-
+    public void testValidateNomatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -65,16 +58,13 @@ public class TermsGroupConfigSerializingTests extends AbstractSerializingTestCas
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("some_other_field", Collections.singletonMap("keyword", fieldCaps));
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [keyword/text] field with name " +
                 "[my_field] in any of the indices matching the index pattern."));
     }
 
-    public void testValidateFieldWrongType() throws IOException {
-
+    public void testValidateFieldWrongType() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -82,17 +72,13 @@ public class TermsGroupConfigSerializingTests extends AbstractSerializingTestCas
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         responseMap.put("my_field", Collections.singletonMap("geo_point", fieldCaps));
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field referenced by a terms group must be a [numeric] or " +
                 "[keyword/text] type, but found [geo_point] for field [my_field]"));
     }
 
-    public void testValidateFieldMatchingNotAggregatable() throws IOException {
-
-
+    public void testValidateFieldMatchingNotAggregatable() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -101,14 +87,12 @@ public class TermsGroupConfigSerializingTests extends AbstractSerializingTestCas
         when(fieldCaps.isAggregatable()).thenReturn(false);
         responseMap.put("my_field", Collections.singletonMap(getRandomType(), fieldCaps));
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not."));
     }
 
-    public void testValidateMatchingField() throws IOException {
+    public void testValidateMatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
         String type = getRandomType();
@@ -118,9 +102,7 @@ public class TermsGroupConfigSerializingTests extends AbstractSerializingTestCas
         when(fieldCaps.isAggregatable()).thenReturn(true);
         responseMap.put("my_field", Collections.singletonMap(type, fieldCaps));
 
-        TermsGroupConfig config = new TermsGroupConfig.Builder()
-                .setFields(Collections.singletonList("my_field"))
-                .build();
+        TermsGroupConfig config = new TermsGroupConfig("my_field");
         config.validateMappings(responseMap, e);
         if (e.validationErrors().size() != 0) {
             fail(e.getMessage());
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
index c7d29451ab3..8b12ae63480 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupIndexer.java
@@ -391,7 +391,6 @@ public abstract class RollupIndexer {
             }
             if (groupConfig.getTerms() != null) {
                 builders.addAll(groupConfig.getTerms().toBuilders());
-                metadata.putAll(groupConfig.getTerms().getMetadata());
             }
         }
 
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java
index d88042188a0..376ba00b063 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java
@@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.HistoGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
+import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 import org.joda.time.DateTimeZone;
 
 import java.util.Arrays;
@@ -297,7 +298,7 @@ public class RollupJobIdentifierUtilTests extends ESTestCase {
         GroupConfig.Builder group2 = ConfigTestHelpers.getGroupConfig();
         group2.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build())
                 .setHisto(null)
-                .setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("bar")).build());
+            .setTerms(new TermsGroupConfig("bar"));
         job2.setGroupConfig(group2.build());
         RollupJobCaps cap2 = new RollupJobCaps(job2.build());
 
@@ -467,7 +468,7 @@ public class RollupJobIdentifierUtilTests extends ESTestCase {
                 .field("bar")
                 .subAggregation(new MaxAggregationBuilder("the_max").field("max_field"))
                 .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field"));
-        
+
         RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo")
                 .setGroupConfig(ConfigTestHelpers.getGroupConfig()
                         .setDateHisto(new DateHistoGroupConfig.Builder()
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
index 60c80a065cb..c2eec92c410 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
@@ -55,6 +55,7 @@ import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps;
 import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
+import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 import org.elasticsearch.xpack.rollup.Rollup;
 import org.hamcrest.core.IsEqual;
 import org.joda.time.DateTimeZone;
@@ -158,7 +159,7 @@ public class SearchActionTests extends ESTestCase {
     public void testTermQuery() {
         RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
         GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build());
+        group.setTerms(new TermsGroupConfig("foo"));
         job.setGroupConfig(group.build());
         RollupJobCaps cap = new RollupJobCaps(job.build());
         Set<RollupJobCaps> caps = new HashSet<>();
@@ -171,7 +172,7 @@ public class SearchActionTests extends ESTestCase {
     public void testTermsQuery() {
         RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
         GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
-        group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build());
+        group.setTerms(new TermsGroupConfig("foo"));
         job.setGroupConfig(group.build());
         RollupJobCaps cap = new RollupJobCaps(job.build());
         Set<RollupJobCaps> caps = new HashSet<>();
@@ -217,7 +218,7 @@ public class SearchActionTests extends ESTestCase {
         RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo");
         GroupConfig.Builder group = ConfigTestHelpers.getGroupConfig();
         group.setDateHisto(new DateHistoGroupConfig.Builder().setField("foo").setInterval(new DateHistogramInterval("1h")).build());
-        group.setTerms(ConfigTestHelpers.getTerms().setFields(Collections.singletonList("foo")).build()).build();
+        group.setTerms(new TermsGroupConfig("foo"));
         job.setGroupConfig(group.build());
         RollupJobCaps cap = new RollupJobCaps(job.build());
         Set<RollupJobCaps> caps = new HashSet<>();
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java
index e465c7883cf..d037bb72e64 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.rollup.config;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.job.DateHistoGroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
 import org.elasticsearch.xpack.core.rollup.job.HistoGroupConfig;
@@ -13,7 +14,6 @@ import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
 import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
-import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.joda.time.DateTimeZone;
 
 import java.util.Arrays;
@@ -59,7 +59,7 @@ public class ConfigTests extends ESTestCase {
 
     public void testNoDateHisto() {
         GroupConfig.Builder groupConfig = new GroupConfig.Builder();
-        groupConfig.setTerms(ConfigTestHelpers.getTerms().build());
+        groupConfig.setTerms(ConfigTestHelpers.randomTermsGroupConfig(random()));
         groupConfig.setHisto(ConfigTestHelpers.getHisto().build());
 
         Exception e = expectThrows(IllegalArgumentException.class, groupConfig::build);
@@ -223,14 +223,9 @@ public class ConfigTests extends ESTestCase {
     }
 
     public void testEmptyTermsField() {
-        TermsGroupConfig.Builder config = ConfigTestHelpers.getTerms();
-        config.setFields(null);
-        Exception e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [fields] must have at least one value."));
-
-        config.setFields(Collections.emptyList());
-        e = expectThrows(IllegalArgumentException.class, config::build);
-        assertThat(e.getMessage(), equalTo("Parameter [fields] must have at least one value."));
+        final String[] fields = randomBoolean() ? new String[0] : null;
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new TermsGroupConfig(fields));
+        assertThat(e.getMessage(), equalTo("Fields must have at least one value"));
     }
 
     public void testNoHeadersInJSON() {
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
index 07ad0af7f1c..29cda6ece75 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/IndexerUtilsTests.java
@@ -431,7 +431,7 @@ public class IndexerUtilsTests extends AggregatorTestCase {
         metricFieldType.setName(metricField);
 
         // Setup the composite agg
-        TermsGroupConfig termsGroupConfig = new TermsGroupConfig.Builder().setFields(Collections.singletonList(valueField)).build();
+        TermsGroupConfig termsGroupConfig = new TermsGroupConfig(valueField);
         CompositeAggregationBuilder compositeBuilder = new CompositeAggregationBuilder(RollupIndexer.AGGREGATION_NAME,
             termsGroupConfig.toBuilders()).size(numDocs*2);
 

From 7d7cddc6ed5646ea277c0591f78fb16ebfb9f4c2 Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Wed, 1 Aug 2018 10:46:02 +0200
Subject: [PATCH 259/260] Watcher: Reenable
 HttpSecretsIntegrationTests#testWebhookAction test (#32456)

The error message mentioned in #30094 does not link to to a cause by the
test itself, as there are still inflight requests according to the
circuit breaker.

I ran this test class 100k times on bare metal and could not reproduce
it. I will reenable the test for now.

Closes #30094
---
 .../watcher/test/integration/HttpSecretsIntegrationTests.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
index 9b7050dc30d..07fb45936e9 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
@@ -165,7 +165,6 @@ public class HttpSecretsIntegrationTests extends AbstractWatcherIntegrationTestC
             is(ApplicableBasicAuth.headerValue(USERNAME, PASSWORD.toCharArray())));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30094")
     public void testWebhookAction() throws Exception {
         WatcherClient watcherClient = watcherClient();
         watcherClient.preparePutWatch("_id")

From 2d87287c0d768256e34a94ddbb1dcd2219eab321 Mon Sep 17 00:00:00 2001
From: Albert Zaharovits <albert.zaharovits@gmail.com>
Date: Wed, 1 Aug 2018 12:07:23 +0300
Subject: [PATCH 260/260] [DOCS] Reloadable Secure Settings (#31713)

Docs on reloadable secure settings for plugins #29135 .
---
 docs/plugins/discovery-ec2.asciidoc           |  5 ++-
 docs/plugins/repository-azure.asciidoc        | 16 ++++++++--
 docs/plugins/repository-gcs.asciidoc          |  9 ++++++
 docs/plugins/repository-s3.asciidoc           | 28 ++++++++++++-----
 docs/reference/setup/secure-settings.asciidoc | 31 +++++++++++++++++++
 5 files changed, 79 insertions(+), 10 deletions(-)

diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
index 5cdc568e162..f79e73fcf55 100644
--- a/docs/plugins/discovery-ec2.asciidoc
+++ b/docs/plugins/discovery-ec2.asciidoc
@@ -37,7 +37,6 @@ bin/elasticsearch-keystore add discovery.ec2.secret_key
 The following are the available discovery settings. All should be prefixed with `discovery.ec2.`.
 Those that must be stored in the keystore are marked as `Secure`.
 
-
 `access_key`::
 
     An ec2 access key. The `secret_key` setting must also be specified. (Secure)
@@ -122,6 +121,10 @@ Defaults to `private_ip`.
     How long the list of hosts is cached to prevent further requests to the AWS API.
     Defaults to `10s`.
 
+*All* secure settings of this plugin are {ref}/secure-settings.html#reloadable-secure-settings[reloadable].
+After you reload the settings, an aws sdk client with the latest settings
+from the keystore will be used.
+
 [IMPORTANT]
 .Binding the network host
 ==============================================
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index 893acf05a4b..13c6a7b62cc 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -11,7 +11,7 @@ include::install_remove.asciidoc[]
 ==== Azure Repository
 
 To enable Azure repositories, you have first to define your azure storage settings as
-{ref}/secure-settings.html[secured settings]:
+{ref}/secure-settings.html[secure settings], before starting up the node:
 
 [source,sh]
 ----------------------------------------------------------------
@@ -20,6 +20,7 @@ bin/elasticsearch-keystore add azure.client.default.key
 ----------------------------------------------------------------
 
 Where `account` is the azure account name and `key` the azure secret key.
+These settings are used by the repository's internal azure client.
 
 Note that you can also define more than one account:
 
@@ -31,7 +32,18 @@ bin/elasticsearch-keystore add azure.client.secondary.account
 bin/elasticsearch-keystore add azure.client.secondary.key
 ----------------------------------------------------------------
 
-`default` is the default account name which will be used by a repository unless you set an explicit one.
+`default` is the default account name which will be used by a repository,
+unless you set an explicit one in the
+<<repository-azure-repository-settings, repository settings>>.
+
+Both `account` and `key` storage settings are
+{ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
+reload the settings, the internal azure clients, which are used to transfer the
+snapshot, will utilize the latest settings from the keystore.
+
+NOTE: In progress snapshot/restore jobs will not be preempted by a *reload*
+of the storage secure settings. They will complete using the client as it was built
+when the operation started.
 
 You can set the client side timeout to use when making any single request. It can be defined globally, per account or both.
 It's not set by default which means that Elasticsearch is using the
diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc
index 84fb47d1761..82923257385 100644
--- a/docs/plugins/repository-gcs.asciidoc
+++ b/docs/plugins/repository-gcs.asciidoc
@@ -112,6 +112,15 @@ PUT _snapshot/my_gcs_repository
 // CONSOLE
 // TEST[skip:we don't have gcs setup while testing this]
 
+The `credentials_file` settings are {ref}/secure-settings.html#reloadable-secure-settings[reloadable].
+After you reload the settings, the internal `gcs` clients, used to transfer the
+snapshot contents, will utilize the latest settings from the keystore.
+
+
+NOTE: In progress snapshot/restore jobs will not be preempted by a *reload*
+of the client's `credentials_file` settings. They will complete using the client
+as it was built when the operation started.
+
 [[repository-gcs-client]]
 ==== Client Settings
 
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 19ead367204..b065951856a 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -35,9 +35,9 @@ PUT _snapshot/my_s3_repository
 ==== Client Settings
 
 The client used to connect to S3 has a number of settings available. Client setting names are of
-the form `s3.client.CLIENT_NAME.SETTING_NAME` and specified inside `elasticsearch.yml`.  The
-default client name looked up by a `s3` repository is called `default`, but can be customized
-with the repository setting `client`. For example:
+the form `s3.client.CLIENT_NAME.SETTING_NAME`. The default client name, which is looked up by
+an `s3` repository, is called `default`. It can be modified using the
+<<repository-s3-repository, repository setting>> `client`. For example:
 
 [source,js]
 ----
@@ -53,8 +53,11 @@ PUT _snapshot/my_s3_repository
 // CONSOLE
 // TEST[skip:we don't have s3 setup while testing this]
 
-Some settings are sensitive and must be stored in the {ref}/secure-settings.html[elasticsearch keystore].
-For example, to use explicit AWS access keys:
+Most client settings are specified inside `elasticsearch.yml`, but some are
+sensitive and must be stored in the {ref}/secure-settings.html[elasticsearch keystore].
+
+For example, before you start the node, run these commands to add AWS access
+key settings to the keystore:
 
 [source,sh]
 ----
@@ -62,8 +65,19 @@ bin/elasticsearch-keystore add s3.client.default.access_key
 bin/elasticsearch-keystore add s3.client.default.secret_key
 ----
 
-The following are the available client settings. Those that must be stored in the keystore
-are marked as `Secure`.
+*All* client secure settings of this plugin are
+{ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
+reload the settings, the internal `s3` clients, used to transfer the snapshot
+contents, will utilize the latest settings from the keystore. Any existing `s3`
+repositories, as well as any newly created ones, will pick up the new values
+stored in the keystore.
+
+NOTE: In progress snapshot/restore tasks will not be preempted by a *reload*
+of the client's secure settings. The task will complete using the client as it
+was built when the operation started.
+
+The following is the list of all the available client settings.
+Those that must be stored in the keystore are marked as `Secure` and are *reloadable*.
 
 `access_key`::
 
diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc
index 1474ba13769..2177440457a 100644
--- a/docs/reference/setup/secure-settings.asciidoc
+++ b/docs/reference/setup/secure-settings.asciidoc
@@ -75,3 +75,34 @@ To remove a setting from the keystore, use the `remove` command:
 bin/elasticsearch-keystore remove the.setting.name.to.remove
 ----------------------------------------------------------------
 
+[float]
+[[reloadable-secure-settings]]
+=== Reloadable secure settings
+
+Just like the settings values in `elasticsearch.yml`, changes to the
+keystore contents are not automatically applied to the running
+elasticsearch node. Re-reading settings requires a node restart.
+However, certain secure settings are marked as *reloadable*. Such settings
+can be re-read and applied on a running node.
+
+The values of all secure settings, *reloadable* or not, must be identical
+across all cluster nodes. After making the desired secure settings changes,
+using the `bin/elasticsearch-keystore add` command, call:
+[source,js]
+----
+POST _nodes/reload_secure_settings
+{
+  "secure_settings_password": ""
+}
+----
+// CONSOLE
+This API will decrypt and re-read the entire keystore, on every cluster node,
+but only the *reloadable* secure settings will be applied. Changes to other
+settings will not go into effect until the next restart. Once the call returns,
+the reload has been completed, meaning that all internal datastructures dependent
+on these settings have been changed. Everything should look as if the settings
+had the new value from the start.
+
+When changing multiple *reloadable* secure settings, modify all of them, on
+each cluster node, and then issue a `reload_secure_settings` call, instead
+of reloading after each modification.