Merge branch 'master' into feature/sql

Original commit: elastic/x-pack-elasticsearch@dd027d9ca5

docs/en/installing-xes.asciidoc

@@ -180,12 +180,16 @@ directory. For example,
 xpack.ssl.key: certs/${node.name}/${node.name}.key <1>
 xpack.ssl.certificate: certs/${node.name}/${node.name}.crt <2>
 xpack.ssl.certificate_authorities: certs/ca/ca.crt <3>
+xpack.security.transport.ssl.enabled: true
+xpack.security.http.ssl.enabled: true <4>
 -----------------------------------------------------------
 <1> If this path does not exist on every node or the file name does not match
 the `node.name` configuration setting, you must specify the full path to the
 node key file.
 <2> Alternatively, specify the full path to the node certificate.
 <3> Alternatively specify the full path to the CA certificate.
+<4> This setting is optional. It enables SSL on the HTTP layer to ensure that
+communication between HTTP clients and the cluster is encrypted. 
 --
 
 .. Start {es}.

docs/en/rest-api/ml/jobresource.asciidoc

@@ -325,14 +325,22 @@ For more information, see
   This value will need to be increased for jobs that are expected to analyze high
   cardinality fields, but the default is set to a relatively small size to ensure
   that high resource usage is a conscious decision. The default value for jobs
-  created in versions earlier than 6.1 is `4096mb`. If you specify a number instead
+  created in versions earlier than 6.1 is `4096mb`.
-  of a string, the units are assumed to be MiB. Specifying a string is recommended
++
-  for clarity. If you specify a byte size unit of `b` or `kb` and the number
+--
-  does not equate to a discrete number of megabytes, it is rounded down to the
+If you specify a number instead of a string, the units are assumed to be MiB.
-  closest MiB. The minimum valid value is 1 MiB. If you specify a value less
+Specifying a string is recommended for clarity. If you specify a byte size unit
-  than 1 MiB, an error occurs. For more information about supported byte size
+of `b` or `kb` and the number does not equate to a discrete number of megabytes,
-  units, see
+it is rounded down to the closest MiB. The minimum valid value is 1 MiB. If you
-  {ref}/common-options.html#byte-units[Byte size units].
+specify a value less than 1 MiB, an error occurs. For more information about
+supported byte size units, see
+{ref}/common-options.html#byte-units[Byte size units].
+
+If your `elasticsearch.yml` file contains an `xpack.ml.max_model_memory_limit`
+setting, an error occurs when you try to create jobs that have
+`model_memory_limit` values greater than that setting. For more information,
+see <<ml-settings>>.
+--
 
 [float]
 [[ml-apimodelplotconfig]]

docs/en/settings/ml-settings.asciidoc

@@ -11,6 +11,17 @@ You do not need to configure any settings to use {ml}. It is enabled by default.
 [[general-ml-settings]]
 ==== General Machine Learning Settings
 
+`node.ml`::
+Set to `true` (default) to identify the node as a _machine learning node_. +
++
+If set to `false` in `elasticsearch.yml`, the node cannot run jobs. If set to
+`true` but `xpack.ml.enabled` is set to `false`, the `node.ml` setting is
+ignored and the node cannot run jobs. If you want to run jobs, there must be at
+least one machine learning node in your cluster. +
++
+IMPORTANT: On dedicated coordinating nodes or dedicated master nodes, disable
+the `node.ml` role.
+
 `xpack.ml.enabled`::
 Set to `true` (default) to enable {ml} on the node. +
 +
@@ -26,16 +37,12 @@ IMPORTANT: If you want to use {ml} features in your cluster, you must have
 `xpack.ml.enabled` set to `true` on all master-eligible nodes. This is the
 default behavior.
 
-`node.ml`::
-Set to `true` (default) to identify the node as a _machine learning node_. +
-+
-If set to `false` in `elasticsearch.yml`, the node cannot run jobs. If set to
-`true` but `xpack.ml.enabled` is set to `false`, the `node.ml` setting is
-ignored and the node cannot run jobs. If you want to run jobs, there must be at
-least one machine learning node in your cluster. +
-+
-IMPORTANT: On dedicated coordinating nodes or dedicated master nodes, disable
-the `node.ml` role.
-
 `xpack.ml.max_open_jobs`::
 The maximum number of jobs that can run on a node. Defaults to `10`.
+
+`xpack.ml.max_model_memory_limit`::
+The maximum `model_memory_limit` property value that can be set for any job on
+this node. If you try to create a job with a `model_memory_limit` property value
+that is greater than this setting value, an error occurs. Existing jobs are not
+affected when you update this setting. For more information about the
+`model_memory_limit` property, see <<ml-apilimits>>.

docs/en/settings/notification-settings.asciidoc

@@ -16,6 +16,11 @@ Slack>>, and <<pagerduty-notification-settings, PagerDuty>>.
 `xpack.watcher.enabled`::
 Set to `false` to disable {watcher} on the node.
 
+`xpack.watcher.history.cleaner_service.enabled`::
+Set to `false` (default) to disable the cleaner service, which removes previous
+versions of {watcher} indices (for example, .watcher-history*) when it
+determines that they are old.
+
 `xpack.http.proxy.host`::
 Specifies the address of the proxy server to use to connect to HTTP services.
 

docs/en/settings/security-settings.asciidoc

@@ -12,6 +12,11 @@ and perform message authentication,
 level security>>, <<realm-settings, configure realms>>,
 and  <<ssl-tls-settings, encrypt communications with SSL>>.
 
+All of these settings can be added to the `elasticsearch.yml` configuration file,
+with the exception of the secure settings, which you add to the {es} keystore.
+For more information about creating and updating the {es} keystore, see
+<<secure-settings>>.
+
 [float]
 [[general-security-settings]]
 ==== General Security Settings
@@ -24,6 +29,7 @@ need to disable {security} in those `kibana.yml` files. For more information
 about disabling {security} in specific {kib} instances, see
 {kibana-ref}/security-settings-kb.html[{kib} Security Settings].
 
+
 [float]
 [[password-security-settings]]
 ==== Default Password Security Settings
@@ -289,9 +295,12 @@ Defaults to `5s` (5 seconds ).
 Path to a PEM encoded file containing the private key.
 
 `ssl.key_passphrase`::
-The passphrase that will be used to decrypt the private key. This value is
+The passphrase that is used to decrypt the private key. This value is
 optional as the key may not be encrypted.
 
+`ssl.secure_key_passphrase` (<<secure-settings,Secure>>)::
+The passphrase that is used to decrypt the private key.
+
 `ssl.certificate`::
 Path to a PEM encoded file containing the certificate (or certificate chain)
 that will be presented to clients when they connect.
@@ -310,20 +319,29 @@ Keystore format, or `PKCS12` to use PKCS#12 files. The default is `jks`.
 `ssl.keystore.password`::
 The password to the keystore.
 
+`ssl.keystore.secure_password` (<<secure-settings,Secure>>)::
+The password to the keystore.
+
 `ssl.keystore.key_password`::
 The password for the key in the keystore. Defaults to the keystore password.
 
+`ssl.keystore.secure_key_password`::
+The password for the key in the keystore. Defaults to the keystore password.
+
 `ssl.truststore.path`::
 The path to the Java Keystore file that contains the certificates to trust.
 `ssl.certificate_authorities` and `ssl.truststore.path` may not be used at the same time.
 
+`ssl.truststore.password`::
+The password to the truststore.
+
+`ssl.truststore.secure_password` (<<secure-settings,Secure>>)::
+The password to the truststore.
+
 `ssl.truststore.type`::
 The format of the keystore file. Should be either `jks` to use the Java
 Keystore format, or `PKCS12` to use PKCS#12 files. The default is `jks`.
 
-`ssl.truststore.password`::
-The password to the truststore.
-
 `ssl.verification_mode`::
 Indicates the type of verification when using `ldaps` to protect against man
 in the middle attacks and certificate forgery. Values are `none`, `certificate`,
@@ -478,13 +496,6 @@ The LDAP Server enforced timeout period for an LDAP search.
 An `s` at the end indicates seconds, or `ms` indicates milliseconds.
 Defaults to `5s` (5 seconds ).
 
-`ssl.key`::
-Path to the PEM encoded file containing the private key.
-
-`ssl.key_passphrase`::
-The passphrase that will be used to decrypt the private key. This value is
-optional as the key may not be encrypted.
-
 `ssl.certificate`::
 Path to a PEM encoded file containing the certificate (or certificate chain)
 that will be presented to clients when they connect.
@@ -492,6 +503,29 @@ that will be presented to clients when they connect.
 `ssl.certificate_authorities`::
 List of paths to PEM encoded certificate files that should be trusted.
 
+`ssl.key`::
+Path to the PEM encoded file containing the private key.
+
+`ssl.key_passphrase`::
+The passphrase that is used to decrypt the private key. This value is
+optional as the key might not be encrypted.
+
+`ssl.secure_key_passphrase` (<<secure-settings,Secure>>)::
+The passphrase that is used to decrypt the private key. This value is
+optional as the key might not be encrypted.
+
+`ssl.keystore.key_password`::
+The password for the key in the keystore. Defaults to the keystore password.
+
+`ssl.keystore.secure_key_password` (<<secure-settings,Secure>>)::
+The password for the key in the keystore. Defaults to the keystore password.
+
+`ssl.keystore.password`::
+The password to the keystore.
+
+`ssl.secure_keystore.password` (<<secure-settings,Secure>>)::
+The password to the keystore.
+
 `ssl.keystore.path`::
 The path to the Java Keystore file that contains a private key and certificate.
 
@@ -499,11 +533,11 @@ The path to the Java Keystore file that contains a private key and certificate.
 The format of the keystore file. Should be either `jks` to use the Java
 Keystore format, or `PKCS12` to use PKCS#12 files. The default is `jks`.
 
-`ssl.keystore.password`::
+`ssl.truststore.password`::
-The password to the keystore.
+The password to the truststore.
 
-`ssl.keystore.key_password`::
+`ssl.truststore.secure_password` (<<secure-settings,Secure>>)::
-The password for the key in the keystore. Defaults to the keystore password.
+The password to the truststore.
 
 `ssl.truststore.path`::
 The path to the Java Keystore file that contains the certificates to trust.
@@ -512,9 +546,6 @@ The path to the Java Keystore file that contains the certificates to trust.
 The format of the truststore file. Should be either `jks` to use the Java
 Keystore format, or `PKCS12` to use PKCS#12 files. The default is `jks`.
 
-`ssl.truststore.password`::
-The password to the truststore.
-
 `ssl.verification_mode`::
 Indicates the type of verification when using `ldaps` to protect against man
 in the middle attacks and certificate forgery. Values are `none`, `certificate`,
@@ -524,7 +555,7 @@ and `full`. Defaults to the value of `xpack.ssl.verification_mode`.
 Supported protocols with versions. Defaults to the value of
 `xpack.ssl.supported_protocols`.
 
-`ssl.cipher_suites`
+`ssl.cipher_suites`::
 Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[
 Java Cryptography Architecture documentation]. Defaults to the value of
 `xpack.ssl.cipher_suites`.
@@ -558,16 +589,19 @@ user's certificate as trusted. Defaults to the trusted certificates configured f
 See the {xpack-ref}/pki-realm.html#pki-ssl-config[SSL settings] section of the PKI realm documentation for more information.
 This setting may not be used with `truststore.path`.
 
-`truststore.path`::
+`truststore.algorithm`::
-The path of a truststore to use. Defaults to the trusted certificates configured for SSL.
+Algorithm for the truststore. Defaults to `SunX509`.
-See the {xpack-ref}/pki-realm.html#pki-ssl-config[SSL settings] section of the PKI realm documentation for more information.
-This setting may not be used with `certificate_authorities`.
 
 `truststore.password`::
 The password for the truststore. Must be provided if `truststore.path` is set.
 
-`truststore.algorithm`::
+`truststore.secure_password` (<<secure-settings,Secure>>)::
-Algorithm for the truststore. Defaults to `SunX509`.
+The password for the truststore.
+
+`truststore.path`::
+The path of a truststore to use. Defaults to the trusted certificates configured for SSL.
+See the {xpack-ref}/pki-realm.html#pki-ssl-config[SSL settings] section of the PKI realm documentation for more information.
+This setting may not be used with `certificate_authorities`.
 
 `files.role_mapping`::
 Specifies the {xpack-ref}/security-files.html[location] of the
@@ -599,7 +633,7 @@ Defaults to `required`.
 Controls the verification of certificates. Valid values are `none`,
 `certificate`, and `full`. Defaults to `full`.
 
-`xpack.ssl.cipher_suites`
+`xpack.ssl.cipher_suites`::
 Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[
 Java Cryptography Architecture documentation]. Defaults to `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256`,
 `TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256`, `TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA`, `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA`,
@@ -628,8 +662,12 @@ When using PEM encoded files, use the following settings:
 Path to the PEM encoded file containing the private key.
 
 `xpack.ssl.key_passphrase`::
-The passphrase that will be used to decrypt the private key. This value is
+The passphrase that is used to decrypt the private key. This value is
-optional as the key may not be encrypted.
+optional as the key might not be encrypted.
+
+`xpack.ssl.secure_key_passphrase` ({<<secure-settings,Secure>>)::
+The passphrase that is used to decrypt the private key. This value is
+optional as the key might not be encrypted.
 
 `xpack.ssl.certificate`::
 Path to a PEM encoded file containing the certificate (or certificate chain)
@@ -650,16 +688,25 @@ Path to the keystore that holds the private key and certificate.
 `xpack.ssl.keystore.password`::
 Password to the keystore.
 
+`xpack.ssl.keystore.secure_password` (<<secure-settings,Secure>>)::
+Password to the keystore.
+
 `xpack.ssl.keystore.key_password`::
 Password for the private key in the keystore. Defaults to the
 same value as `xpack.ssl.keystore.password`.
 
+`xpack.ssl.keystore.secure_key_password` (<<secure-settings,Secure>>)::
+Password for the private key in the keystore.
+
 `xpack.ssl.truststore.path`::
 Path to the truststore file.
 
 `xpack.ssl.truststore.password`::
 Password to the truststore.
 
+`xpack.ssl.truststore.secure_password` (<<secure-settings,Secure>>)::
+Password to the truststore.
+
 [float]
 ===== PKCS#12 Files
 
@@ -676,10 +723,16 @@ Set this to `PKCS12`.
 `xpack.ssl.keystore.password`::
 Password to the PKCS#12 file.
 
+`xpack.ssl.keystore.secure_password` (<<secure-settings,Secure>>)::
+Password to the PKCS#12 file.
+
 `xpack.ssl.keystore.key_password`::
 Password for the private key in the PKCS12 file.
 Defaults to the same value as `xpack.ssl.keystore.password`.
 
+`xpack.ssl.keystore.secure_key_password` (<<secure-settings,Secure>>)::
+Password for the private key in the PKCS12 file.
+
 `xpack.ssl.truststore.path`::
 Path to the truststore file.
 
@@ -689,6 +742,9 @@ Set this to `PKCS12`.
 `xpack.ssl.truststore.password`::
 Password to the truststore.
 
+`xpack.ssl.truststore.secure_password` (<<secure-settings,Secure>>)::
+Password to the truststore.
+
 [[http-tls-ssl-settings]]
 :ssl-prefix:             xpack.security.http
 :component:              HTTP

docs/en/settings/ssl-settings.asciidoc

@@ -63,8 +63,12 @@ When using PEM encoded files, use the following settings:
 Path to a PEM encoded file containing the private key.
 
 +{ssl-prefix}.ssl.key_passphrase+::
-The passphrase that will be used to decrypt the private key. This value is
+The passphrase that is used to decrypt the private key. This value is optional
-optional as the key may not be encrypted.
+as the key might not be encrypted.
+
++{ssl-prefix}.ssl.secure_key_passphrase+ (<<secure-settings,Secure>>)::
+The passphrase that is used to decrypt the private key. This value is optional
+as the key might not be encrypted.
 
 +{ssl-prefix}.ssl.certificate+::
 Path to a PEM encoded file containing the certificate (or certificate chain)
@@ -84,16 +88,25 @@ Path to the keystore that holds the private key and certificate.
 +{ssl-prefix}.ssl.keystore.password+::
 Password to the keystore.
 
++{ssl-prefix}.ssl.keystore.secure_password` (<<secure-settings,Secure>>)::
+Password to the keystore.
+
 +{ssl-prefix}.ssl.keystore.key_password+::
 Password for the private key in the keystore. Defaults to the
 same value as +{ssl-prefix}.ssl.keystore.password+.
 
++{ssl-prefix}.ssl.keystore.secure_key_password+ (<<secure-settings,Secure>>)::
+Password for the private key in the keystore.
+
 +{ssl-prefix}.ssl.truststore.path+::
 Path to the truststore file.
 
 +{ssl-prefix}.ssl.truststore.password+::
 Password to the truststore.
 
++{ssl-prefix}.ssl.truststore.secure_password+ (<<secure-settings,Secure>>)::
+Password to the truststore.
+
 ===== PKCS#12 Files
 
 {security} can be configured to use PKCS#12 container files (`.p12` or `.pfx` files)
@@ -110,10 +123,16 @@ Set this to `PKCS12` to indicate that the keystore is a PKCS#12 file.
 +{ssl-prefix}.ssl.keystore.password+::
 Password to the PKCS#12 file.
 
++{ssl-prefix}.ssl.keystore.secure_password+ (<<secure-settings,Secure>>)::
+Password to the PKCS#12 file.
+
 +{ssl-prefix}.ssl.keystore.key_password+::
 Password for the private key stored in the PKCS#12 file.
 Defaults to the same value as +{ssl-prefix}.ssl.keystore.password+.
 
++{ssl-prefix}.ssl.keystore.secure_key_password+ (<<secure-settings,Secure>>)::
+Password for the private key stored in the PKCS#12 file.
+
 +{ssl-prefix}.ssl.truststore.path+::
 Path to the PKCS#12 file that holds the certificates to be trusted.
 
@@ -122,3 +141,6 @@ Set this to `PKCS12` to indicate that the truststore is a PKCS#12 file.
 
 +{ssl-prefix}.ssl.truststore.password+::
 Password to the PKCS#12 file.
+
++{ssl-prefix}.ssl.truststore.secure_password+ (<<secure-settings,Secure>>)::
+Password to the PKCS#12 file.

docs/en/watcher/input/http.asciidoc

@@ -73,7 +73,7 @@ Stats] API and enables the `human` attribute:
   "http" : {
     "request" : {
       "host" : "host.domain",
-      "port" : "9200",
+      "port" : 9200,
       "path" : "/_cluster/stats",
       "params" : {
         "human" : "true" <1>
@@ -98,7 +98,7 @@ a username and password to access `myservice`:
   "http" : {
     "request" : {
       "host" : "host.domain",
-      "port" : "9200",
+      "port" : 9200,
       "path" : "/myservice",
       "auth" : {
         "basic" : {

plugin/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java

@@ -830,7 +830,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
 
     Settings customAuditIndexSettings(Settings nodeSettings) {
         Settings newSettings = Settings.builder()
-                .put(INDEX_SETTINGS.get(nodeSettings))
+                .put(INDEX_SETTINGS.get(nodeSettings), false)
                 .build();
         if (newSettings.names().isEmpty()) {
             return Settings.EMPTY;

plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java

@@ -17,24 +17,21 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.apache.lucene.search.join.ToChildBlockJoinQuery;
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.SparseFixedBitSet;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.FilterClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -46,7 +43,6 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.GeoShapeQueryBuilder;
-import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -139,10 +135,17 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
                         QueryBuilder queryBuilder = queryShardContext.parseInnerQueryBuilder(parser);
                         verifyRoleQuery(queryBuilder);
                         failIfQueryUsesClient(queryBuilder, queryShardContext);
-                        ParsedQuery parsedQuery = queryShardContext.toFilter(queryBuilder);
+                        Query roleQuery = queryShardContext.toFilter(queryBuilder).query();
-                        filter.add(parsedQuery.query(), SHOULD);
+                        filter.add(roleQuery, SHOULD);
+                        if (queryShardContext.getMapperService().hasNested()) {
+                            // If access is allowed on root doc then also access is allowed on all nested docs of that root document:
+                            BitSetProducer rootDocs = queryShardContext.bitsetFilter(Queries.newNonNestedFilter());
+                            ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs);
+                            filter.add(includeNestedDocs, SHOULD);
                         }
                     }
+                }
+
                 // at least one of the queries should match
                 filter.setMinimumNumberShouldMatch(1);
                 reader = DocumentSubsetReader.wrap(reader, bitsetFilterCache, new ConstantScoreQuery(filter.build()));

plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java

@@ -21,6 +21,7 @@ import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.upgrade.Upgrade;
 import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
+import org.elasticsearch.xpack.watcher.support.WatcherIndexTemplateRegistry;
 import org.elasticsearch.xpack.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.watch.WatchStoreUtils;
 
@@ -76,6 +77,13 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
             return;
         }
 
+        // ensure that templates are existing before starting watcher
+        // the watcher index template registry is independent from watcher being started or stopped
+        if (WatcherIndexTemplateRegistry.validate(state) == false) {
+            logger.debug("not starting watcher, watcher templates are missing in the cluster state");
+            return;
+        }
+
         if (watcherService.validate(state)) {
             logger.trace("starting... (based on cluster state version [{}]) (manual [{}])", state.getVersion(), manual);
             try {

plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java

@@ -127,6 +127,12 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
         });
     }
 
+    public static boolean validate(ClusterState state) {
+        return state.getMetaData().getTemplates().containsKey(HISTORY_TEMPLATE_NAME) &&
+                state.getMetaData().getTemplates().containsKey(TRIGGERED_TEMPLATE_NAME) &&
+                state.getMetaData().getTemplates().containsKey(WATCHES_TEMPLATE_NAME);
+    }
+
     public static class TemplateConfig {
 
         private final String templateName;

plugin/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java

@@ -23,6 +23,8 @@ import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.IndicesRequestCache;
@@ -37,8 +39,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortMode;
 import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.xpack.XPackPlugin;
@@ -52,6 +52,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
@@ -658,7 +659,7 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
     }
 
     public void testParentChild_joinField() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
+        XContentBuilder mapping = jsonBuilder().startObject()
                 .startObject("properties")
                     .startObject("join_field")
                         .field("type", "join")
@@ -903,4 +904,44 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
         assertThat(client().prepareGet("test", "type", "1").get().getSource().get("field1").toString(), equalTo("value3"));
     }
 
+    public void testNestedInnerHits() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "nested_field", "type=nested")
+        );
+        client().prepareIndex("test", "type1", "1")
+                .setSource(jsonBuilder().startObject()
+                            .field("field1", "value1")
+                            .startArray("nested_field")
+                                .startObject()
+                                    .field("field2", "value2")
+                                .endObject()
+                            .endArray()
+                        .endObject())
+                .get();
+        client().prepareIndex("test", "type1", "2")
+                .setSource(jsonBuilder().startObject()
+                            .field("field1", "value2")
+                            .startArray("nested_field")
+                                .startObject()
+                                    .field("field2", "value2")
+                                .endObject()
+                            .endArray()
+                        .endObject())
+                .get();
+        refresh("test");
+
+        SearchResponse response = client()
+                .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                .prepareSearch("test")
+                .setQuery(QueryBuilders.nestedQuery("nested_field", QueryBuilders.termQuery("nested_field.field2", "value2"),
+                        ScoreMode.None).innerHit(new InnerHitBuilder()))
+                .get();
+        assertHitCount(response, 1);
+        assertSearchHits(response, "1");
+        assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getId(), equalTo("1"));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getNestedIdentity().getOffset(), equalTo(0));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getSourceAsString(),
+                equalTo("{\"field2\":\"value2\"}"));
+    }
+
 }

plugin/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java

@@ -79,7 +79,7 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
             null, null, null, null, null, null));
         List<NodeStats> nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null,
             null, null, null, null, null, null, null, null,
-            null, null, null));
+            null, null, null, null));
         IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(Settings.EMPTY);
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false,
             true, true);

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java

@@ -372,6 +372,6 @@ public class NodeStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestCa
                                                                 emptySet(),
                                                                 Version.V_6_0_0_beta1);
 
-        return new NodeStats(discoveryNode, no, indices, os, process, jvm, threadPool, fs, null, null, null, null, null, null);
+        return new NodeStats(discoveryNode, no, indices, os, process, jvm, threadPool, fs, null, null, null, null, null, null, null);
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrailTests.java

@@ -22,6 +22,8 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.plugins.Plugin;
@@ -276,11 +278,11 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase {
         return remoteIndexing ? remoteCluster.client() : client();
     }
 
-    private void initialize() throws IOException, InterruptedException {
+    private void initialize() throws Exception {
         initialize(null, null);
     }
 
-    private void initialize(String[] includes, String[] excludes) throws IOException, InterruptedException {
+    private void initialize(String[] includes, String[] excludes) throws Exception {
         rollover = randomFrom(HOURLY, DAILY, WEEKLY, MONTHLY);
         numReplicas = numberOfReplicas();
         numShards = numberOfShards();
@@ -289,8 +291,18 @@ public class IndexAuditTrailTests extends SecurityIntegTestCase {
         if (remoteIndexing) {
             builder.put(remoteSettings);
         }
+        builder.put(settings(rollover, includes, excludes));
+        // IndexAuditTrail should ignore secure settings
+        // they are merged on the master node creating the audit index
+        if (randomBoolean()) {
+            MockSecureSettings ignored = new MockSecureSettings();
+            if (randomBoolean()) {
+                ignored.setString(KeyStoreWrapper.SEED_SETTING.getKey(), "non-empty-secure-settings");
+            }
+            builder.setSecureSettings(ignored);
+        }
+        Settings settings = builder.build();
 
-        Settings settings = builder.put(settings(rollover, includes, excludes)).build();
         logger.info("--> settings: [{}]", settings);
         DiscoveryNode localNode = mock(DiscoveryNode.class);
         when(localNode.getHostAddress()).thenReturn(remoteAddress.getAddress());

plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java

@@ -13,6 +13,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -30,6 +31,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
+import org.elasticsearch.xpack.watcher.support.WatcherIndexTemplateRegistry;
 import org.elasticsearch.xpack.watcher.watch.Watch;
 import org.junit.Before;
 import org.mockito.stubbing.Answer;
@@ -82,6 +84,11 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
 
         IndexRoutingTable watchRoutingTable = IndexRoutingTable.builder(new Index(Watch.INDEX, "foo")).build();
         ClusterState clusterState = ClusterState.builder(new ClusterName("my-cluster"))
+                .metaData(MetaData.builder()
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME))
+                        .build())
                 .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1")))
                 .routingTable(RoutingTable.builder().add(watchRoutingTable).build())
                 .build();
@@ -114,6 +121,11 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
         ClusterState clusterState = ClusterState.builder(new ClusterName("my-cluster"))
                 .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1")))
                 .routingTable(RoutingTable.builder().add(watchRoutingTable).build())
+                .metaData(MetaData.builder()
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME))
+                        .build())
                 .build();
 
         when(watcherService.validate(clusterState)).thenReturn(true);
@@ -138,6 +150,11 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
         // no change, keep going
         clusterState  = ClusterState.builder(new ClusterName("my-cluster"))
                 .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1")))
+                .metaData(MetaData.builder()
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME))
+                        .build())
                 .build();
         when(watcherService.state()).thenReturn(WatcherState.STARTED);
         lifeCycleService.clusterChanged(new ClusterChangedEvent("any", clusterState, clusterState));
@@ -146,6 +163,11 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
 
         ClusterState previousClusterState = ClusterState.builder(new ClusterName("my-cluster"))
                 .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1")))
+                .metaData(MetaData.builder()
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME))
+                        .build())
                 .build();
         when(watcherService.validate(clusterState)).thenReturn(true);
         when(watcherService.state()).thenReturn(WatcherState.STOPPED);
@@ -413,7 +435,14 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
                 .add(newNode("oldNode", VersionUtils.randomVersionBetween(random(), Version.V_5_5_0, Version.V_6_0_0_alpha2)))
                 .build();
 
-        ClusterState state = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).build();
+        ClusterState state = ClusterState.builder(new ClusterName("my-cluster"))
+                .nodes(nodes)
+                .metaData(MetaData.builder()
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME))
+                        .put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME))
+                        .build())
+                .build();
         when(watcherService.validate(eq(state))).thenReturn(true);
         when(watcherService.state()).thenReturn(WatcherState.STOPPED);
 
@@ -421,6 +450,36 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
         verify(watcherService).start(any(ClusterState.class));
     }
 
+    public void testWatcherServiceDoesNotStartIfIndexTemplatesAreMissing() throws Exception {
+        DiscoveryNodes nodes = new DiscoveryNodes.Builder()
+                .masterNodeId("node_1").localNodeId("node_1")
+                .add(newNode("node_1"))
+                .build();
+
+        MetaData.Builder metaDataBuilder = MetaData.builder();
+        boolean isHistoryTemplateAdded = randomBoolean();
+        if (isHistoryTemplateAdded) {
+            metaDataBuilder.put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME));
+        }
+        boolean isTriggeredTemplateAdded = randomBoolean();
+        if (isTriggeredTemplateAdded) {
+            metaDataBuilder.put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME));
+        }
+        boolean isWatchesTemplateAdded = randomBoolean();
+        if (isWatchesTemplateAdded) {
+            // ensure not all templates are added, otherwise life cycle service would start
+            if ((isHistoryTemplateAdded || isTriggeredTemplateAdded) == false) {
+                metaDataBuilder.put(IndexTemplateMetaData.builder(WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME));
+            }
+        }
+        ClusterState state = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).metaData(metaDataBuilder).build();
+        when(watcherService.validate(eq(state))).thenReturn(true);
+        when(watcherService.state()).thenReturn(WatcherState.STOPPED);
+
+        lifeCycleService.clusterChanged(new ClusterChangedEvent("any", state, state));
+        verify(watcherService, times(0)).start(any(ClusterState.class));
+    }
+
     private static DiscoveryNode newNode(String nodeName) {
         return newNode(nodeName, Version.CURRENT);
     }

plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java

@@ -13,13 +13,13 @@ import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.IndicesAdminClient;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -36,6 +36,7 @@ import java.util.List;
 
 import static org.elasticsearch.mock.orig.Mockito.verify;
 import static org.elasticsearch.mock.orig.Mockito.when;
+import static org.hamcrest.Matchers.is;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Matchers.eq;
@@ -51,8 +52,6 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
 
     @Before
     public void createRegistryAndClient() {
-        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Collections.emptySet());
-
         ThreadPool threadPool = mock(ThreadPool.class);
         when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
         when(threadPool.generic()).thenReturn(EsExecutors.newDirectExecutorService());
@@ -108,6 +107,25 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
         verify(client, times(4)).execute(anyObject(), argumentCaptor.capture(), anyObject());
     }
 
+    public void testThatTemplatesExist() {
+        assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history")), is(false));
+        assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history", ".triggered_watches", ".watches")),
+                is(false));
+        assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME,
+                ".triggered_watches", ".watches")), is(true));
+        assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistry.HISTORY_TEMPLATE_NAME,
+                ".triggered_watches", ".watches", "whatever", "else")), is(true));
+    }
+
+    private ClusterState createClusterState(String ... existingTemplates) {
+        MetaData.Builder metaDataBuilder = MetaData.builder();
+        for (String templateName : existingTemplates) {
+            metaDataBuilder.put(IndexTemplateMetaData.builder(templateName));
+        }
+
+        return ClusterState.builder(new ClusterName("foo")).metaData(metaDataBuilder.build()).build();
+    }
+
     private static class TestPutIndexTemplateResponse extends PutIndexTemplateResponse {
         TestPutIndexTemplateResponse(boolean acknowledged) {
             super(acknowledged);