Merge branch 'master' into license-checking/security/better-messaging

Original commit: elastic/x-pack-elasticsearch@fc239142e2

elasticsearch/qa/smoke-test-security-with-mustache/build.gradle

@@ -0,0 +1,27 @@
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
+  testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
+}
+
+integTest {
+  cluster {
+    plugin ':x-plugins:elasticsearch:x-pack'
+    setting 'xpack.watcher.enabled', 'false'
+    setting 'xpack.monitoring.enabled', 'false'
+    setting 'path.scripts', "${project.buildDir}/resources/test/templates"
+    setupCommand 'setupDummyUser',
+            'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'superuser'
+    waitCondition = { node, ant ->
+      File tmpFile = new File(node.cwd, 'wait.success')
+      ant.get(src: "http://${node.httpUri()}",
+              dest: tmpFile.toString(),
+              username: 'test_admin',
+              password: 'changeme',
+              ignoreerrors: true,
+              retries: 10)
+      return tmpFile.exists()
+    }
+  }
+}

elasticsearch/qa/smoke-test-security-with-mustache/src/test/java/org/elasticsearch/smoketest/RestIT.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.smoketest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+import org.elasticsearch.xpack.security.authc.support.SecuredString;
+
+import java.io.IOException;
+
+import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+public class RestIT extends ESRestTestCase {
+
+    private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("test_admin", new SecuredString("changeme".toCharArray()));
+
+    public RestIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder()
+                .put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE)
+                .build();
+    }
+}

elasticsearch/qa/smoke-test-security-with-mustache/src/test/resources/rest-api-spec/test/templated_role_query/10_templated_role_query.yaml

@@ -0,0 +1,191 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  - do:
+      xpack.security.put_user:
+        username: "inline_template_user"
+        body:  >
+            {
+              "password": "changeme",
+              "roles" : [ "inline_template_role" ]
+            }
+  - do:
+      xpack.security.put_user:
+        username: "stored_template_user"
+        body:  >
+            {
+              "password": "changeme",
+              "roles" : [ "stored_template_role" ]
+            }
+
+  - do:
+      xpack.security.put_user:
+        username: "file_template_user"
+        body:  >
+            {
+              "password": "changeme",
+              "roles" : [ "file_template_role" ]
+            }
+
+  - do:
+      xpack.security.put_role:
+        name: "inline_template_role"
+        body:  >
+            {
+              "indices": [
+                {
+                  "names": "foobar",
+                  "privileges": ["all"],
+                  "query" : {
+                    "template" : {
+                      "inline" : {
+                        "term" : { "username" : "{{_user.username}}" }
+                      }
+                    }
+                  }
+                }
+              ]
+            }
+
+  - do:
+      xpack.security.put_role:
+        name: "stored_template_role"
+        body:  >
+            {
+              "indices": [
+                {
+                  "names": "foobar",
+                  "privileges": ["all"],
+                  "query" : {
+                    "template" : {
+                      "id" : "1"
+                    }
+                  }
+                }
+              ]
+            }
+
+  - do:
+      xpack.security.put_role:
+        name: "file_template_role"
+        body:  >
+            {
+              "indices": [
+                {
+                  "names": "foobar",
+                  "privileges": ["all"],
+                  "query" : {
+                    "template" : {
+                      "file" : "query"
+                    }
+                  }
+                }
+              ]
+            }
+
+  - do:
+      put_template:
+        id: "1"
+        body: >
+            {
+              "term" : {
+                "username" : "{{_user.username}}"
+              }
+            }
+
+  - do:
+      index:
+        index: foobar
+        type: type
+        id: 1
+        body:  >
+          {
+            "username": "inline_template_user"
+          }
+  - do:
+      index:
+        index: foobar
+        type: type
+        id: 2
+        body:  >
+          {
+            "username": "stored_template_user"
+          }
+  - do:
+      index:
+        index: foobar
+        type: type
+        id: 3
+        body:  >
+          {
+            "username": "file_template_user"
+          }
+
+  - do:
+      indices.refresh: {}
+
+---
+teardown:
+  - do:
+      xpack.security.delete_user:
+        username: "inline_template_user"
+        ignore: 404
+  - do:
+      xpack.security.delete_user:
+        username: "stored_template_user"
+        ignore: 404
+  - do:
+      xpack.security.delete_user:
+        username: "file_template_user"
+        ignore: 404
+  - do:
+      xpack.security.delete_role:
+        name: "inline_template_role"
+        ignore: 404
+  - do:
+      xpack.security.delete_role:
+        name: "stored_template_role"
+        ignore: 404
+  - do:
+      xpack.security.delete_role:
+        name: "file_template_role"
+        ignore: 404
+
+---
+"Test inline template":
+  - do:
+      headers:
+        Authorization: "Basic aW5saW5lX3RlbXBsYXRlX3VzZXI6Y2hhbmdlbWU="
+      search:
+        index: foobar
+        body: { "query" : { "match_all" : {} } }
+  - match: { hits.total: 1}
+  - match: { hits.hits.0._source.username: inline_template_user}
+
+---
+"Test stored template":
+  - do:
+      headers:
+        Authorization: "Basic c3RvcmVkX3RlbXBsYXRlX3VzZXI6Y2hhbmdlbWU="
+      search:
+        index: foobar
+        body: { "query" : { "match_all" : {} } }
+  - match: { hits.total: 1}
+  - match: { hits.hits.0._source.username: stored_template_user}
+
+---
+"Test file template":
+  - do:
+      headers:
+        Authorization: "Basic ZmlsZV90ZW1wbGF0ZV91c2VyOmNoYW5nZW1l"
+      search:
+        index: foobar
+        body: { "query" : { "match_all" : {} } }
+  - match: { hits.total: 1}
+  - match: { hits.hits.0._source.username: file_template_user}

elasticsearch/qa/smoke-test-security-with-mustache/src/test/resources/rest-api-spec/test/templated_role_query/20_small_users_one_index.yaml

@@ -0,0 +1,198 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      indices.create:
+        index: shared_logs
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "processors": [
+             {
+                "set_security_user" : {
+                  "field" : "user"
+                }
+              }
+            ]
+          }
+  - do:
+      xpack.security.put_user:
+        username: "joe"
+        body:  >
+            {
+              "password": "changeme",
+              "roles" : [ "small_companies_role" ],
+              "metadata" : {
+                "customer_id" : "1"
+              }
+            }
+  - do:
+      xpack.security.put_user:
+        username: "john"
+        body:  >
+            {
+              "password": "changeme",
+              "roles" : [ "small_companies_role" ],
+              "metadata" : {
+                "customer_id" : "2"
+              }
+            }
+
+---
+teardown:
+  - do:
+      xpack.security.delete_user:
+        username: "joe"
+        ignore: 404
+  - do:
+      xpack.security.delete_user:
+        username: "john"
+        ignore: 404
+  - do:
+      xpack.security.delete_role:
+        name: "small_companies_role"
+        ignore: 404
+
+---
+"Test shared index seperating user by using DLS role query with user's username":
+  - do:
+      xpack.security.put_role:
+        name: "small_companies_role"
+        body:  >
+            {
+              "indices": [
+                {
+                  "names": "shared_logs",
+                  "privileges": ["read", "create"],
+                  "query" : {
+                    "template" : {
+                      "inline" : {
+                        "term" : { "user.username" : "{{_user.username}}" }
+                      }
+                    }
+                  }
+                }
+              ]
+            }
+
+  - do:
+      headers:
+        Authorization: "Basic am9lOmNoYW5nZW1l"
+      index:
+        index: shared_logs
+        type: type
+        pipeline: "my_pipeline"
+        body:  >
+          {
+            "log": "Joe's first log entry"
+          }
+  - do:
+      headers:
+        Authorization: "Basic am9objpjaGFuZ2VtZQ=="
+      index:
+        index: shared_logs
+        type: type
+        pipeline: "my_pipeline"
+        body:  >
+          {
+            "log": "John's first log entry"
+          }
+
+  - do:
+      indices.refresh: {}
+
+  # Joe searches:
+  - do:
+      headers:
+        Authorization: "Basic am9lOmNoYW5nZW1l"
+      search:
+        index: shared_logs
+        body: { "query" : { "match_all" : {} } }
+  - match: { hits.total: 1}
+  - match: { hits.hits.0._source.user.username: joe}
+
+  # John searches:
+  - do:
+      headers:
+        Authorization: "Basic am9objpjaGFuZ2VtZQ=="
+      search:
+        index: shared_logs
+        body: { "query" : { "match_all" : {} } }
+  - match: { hits.total: 1}
+  - match: { hits.hits.0._source.user.username: john}
+
+---
+"Test shared index seperating user by using DLS role query with user's metadata":
+  - do:
+      xpack.security.put_role:
+        name: "small_companies_role"
+        body:  >
+            {
+              "indices": [
+                {
+                  "names": "shared_logs",
+                  "privileges": ["read", "create"],
+                  "query" : {
+                    "template" : {
+                      "inline" : {
+                        "term" : { "user.metadata.customer_id" : "{{_user.metadata.customer_id}}" }
+                      }
+                    }
+                  }
+                }
+              ]
+            }
+
+  - do:
+      headers:
+        Authorization: "Basic am9lOmNoYW5nZW1l"
+      index:
+        index: shared_logs
+        type: type
+        pipeline: "my_pipeline"
+        body:  >
+          {
+            "log": "Joe's first log entry"
+          }
+  - do:
+      headers:
+        Authorization: "Basic am9objpjaGFuZ2VtZQ=="
+      index:
+        index: shared_logs
+        type: type
+        pipeline: "my_pipeline"
+        body:  >
+          {
+            "log": "John's first log entry"
+          }
+
+  - do:
+      indices.refresh: {}
+
+  # Joe searches:
+  - do:
+      headers:
+        Authorization: "Basic am9lOmNoYW5nZW1l"
+      search:
+        index: shared_logs
+        body: { "query" : { "match_all" : {} } }
+  - match: { hits.total: 1}
+  - match: { hits.hits.0._source.user.username: joe}
+
+  # John searches:
+  - do:
+      headers:
+        Authorization: "Basic am9objpjaGFuZ2VtZQ=="
+      search:
+        index: shared_logs
+        body: { "query" : { "match_all" : {} } }
+  - match: { hits.total: 1}
+  - match: { hits.hits.0._source.user.username: john}

elasticsearch/qa/smoke-test-security-with-mustache/src/test/resources/templates/query.mustache

@@ -0,0 +1,5 @@
+{
+  "term" : {
+    "username" : "{{_user.username}}"
+  }
+}

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -430,7 +430,7 @@ public class Security implements ActionPlugin, IngestPlugin {
             module.setSearcherWrapper((indexService) -> new SecurityIndexSearcherWrapper(indexService.getIndexSettings(),
                     indexService.newQueryShardContext(), indexService.mapperService(),
                     indexService.cache().bitsetFilterCache(), indexService.getIndexServices().getThreadPool().getThreadContext(),
-                    securityLicenseState));
+                    securityLicenseState, indexService.getIndexServices().getScriptService()));
         }
         if (transportClientMode == false) {
             /*  We need to forcefully overwrite the query cache implementation to use security's opt out query cache implementation.

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java

@@ -22,7 +22,9 @@ import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.SparseFixedBitSet;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
@@ -43,16 +45,24 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.xpack.security.authc.Authentication;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader;
 import org.elasticsearch.xpack.security.SecurityLicenseState;
 import org.elasticsearch.xpack.security.support.Exceptions;
+import org.elasticsearch.xpack.security.user.User;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
@@ -78,10 +88,13 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
     private final SecurityLicenseState securityLicenseState;
     private final ThreadContext threadContext;
     private final ESLogger logger;
+    private final ScriptService scriptService;
 
     public SecurityIndexSearcherWrapper(IndexSettings indexSettings, QueryShardContext queryShardContext,
                                         MapperService mapperService, BitsetFilterCache bitsetFilterCache,
-                                        ThreadContext threadContext, SecurityLicenseState securityLicenseState) {
+                                        ThreadContext threadContext, SecurityLicenseState securityLicenseState,
+                                        ScriptService scriptService) {
+        this.scriptService = scriptService;
         this.logger = Loggers.getLogger(getClass(), indexSettings.getSettings());
         this.mapperService = mapperService;
         this.queryShardContext = queryShardContext;
@@ -124,6 +137,7 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
                 BooleanQuery.Builder filter = new BooleanQuery.Builder();
                 for (BytesReference bytesReference : permissions.getQueries()) {
                     QueryShardContext queryShardContext = copyQueryShardContext(this.queryShardContext);
+                    bytesReference = evaluateTemplate(bytesReference);
                     try (XContentParser parser = XContentFactory.xContent(bytesReference).createParser(bytesReference)) {
                         Optional<QueryBuilder> queryBuilder = queryShardContext.newParseContext(parser).parseInnerQueryBuilder();
                         if (queryBuilder.isPresent()) {
@@ -262,6 +276,45 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
         }
     }
 
+    BytesReference evaluateTemplate(BytesReference querySource) throws IOException {
+        try (XContentParser parser = XContentFactory.xContent(querySource).createParser(querySource)) {
+            XContentParser.Token token = parser.nextToken();
+            if (token != XContentParser.Token.START_OBJECT) {
+                throw new ElasticsearchParseException("Unexpected token [" + token + "]");
+            }
+            token = parser.nextToken();
+            if (token != XContentParser.Token.FIELD_NAME) {
+                throw new ElasticsearchParseException("Unexpected token [" + token + "]");
+            }
+            if ("template".equals(parser.currentName())) {
+                token = parser.nextToken();
+                if (token != XContentParser.Token.START_OBJECT) {
+                    throw new ElasticsearchParseException("Unexpected token [" + token + "]");
+                }
+                Script script = Script.parse(parser, ParseFieldMatcher.EMPTY);
+                // Add the user details to the params
+                Map<String, Object> params = new HashMap<>();
+                if (script.getParams() != null) {
+                    params.putAll(script.getParams());
+                }
+                User user = getUser();
+                Map<String, Object> userModel = new HashMap<>();
+                userModel.put("username", user.principal());
+                userModel.put("full_name", user.fullName());
+                userModel.put("email", user.email());
+                userModel.put("roles", Arrays.asList(user.roles()));
+                userModel.put("metadata", Collections.unmodifiableMap(user.metadata()));
+                params.put("_user", userModel);
+                // Always enforce mustache script lang:
+                script = new Script(script.getScript(), script.getType(), "mustache", params, script.getContentType());
+                ExecutableScript executable = scriptService.executable(script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
+                return (BytesReference) executable.run();
+            } else {
+                return querySource;
+            }
+        }
+    }
+
     protected IndicesAccessControl getIndicesAccessControl() {
         IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationService.INDICES_PERMISSIONS_KEY);
         if (indicesAccessControl == null) {
@@ -269,4 +322,9 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
         }
         return indicesAccessControl;
     }
+
+    protected User getUser(){
+        Authentication authentication = Authentication.getAuthentication(threadContext);
+        return authentication.getUser();
+    }
 }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java

@@ -34,6 +34,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.xpack.security.SecurityLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -55,13 +56,14 @@ public class SecurityIndexSearcherWrapperIntegrationTests extends ESTestCase {
     public void testDLS() throws Exception {
         ShardId shardId = new ShardId("_index", "_na_", 0);
         MapperService mapperService = mock(MapperService.class);
+        ScriptService  scriptService = mock(ScriptService.class);
         when(mapperService.docMappers(anyBoolean())).thenReturn(Collections.emptyList());
         when(mapperService.simpleMatchToIndexNames(anyString()))
                 .then(invocationOnMock -> Collections.singletonList((String) invocationOnMock.getArguments()[0]));
 
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, null,
-                singleton(new BytesArray("{}")));
+                singleton(new BytesArray("{\"match_all\" : {}}")));
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY);
         QueryShardContext queryShardContext = mock(QueryShardContext.class);
         QueryParseContext queryParseContext = mock(QueryParseContext.class);
@@ -79,7 +81,7 @@ public class SecurityIndexSearcherWrapperIntegrationTests extends ESTestCase {
         SecurityLicenseState licenseState = mock(SecurityLicenseState.class);
         when(licenseState.documentAndFieldLevelSecurityEnabled()).thenReturn(true);
         SecurityIndexSearcherWrapper wrapper = new SecurityIndexSearcherWrapper(indexSettings, queryShardContext, mapperService,
-                bitsetFilterCache, threadContext, licenseState) {
+                bitsetFilterCache, threadContext, licenseState, scriptService) {
 
             @Override
             protected QueryShardContext copyQueryShardContext(QueryShardContext context) {
@@ -140,7 +142,7 @@ public class SecurityIndexSearcherWrapperIntegrationTests extends ESTestCase {
             ParsedQuery parsedQuery = new ParsedQuery(new TermQuery(new Term("field", values[i])));
             when(queryShardContext.newParseContext(any(XContentParser.class))).thenReturn(queryParseContext);
             when(queryParseContext.parseInnerQueryBuilder())
-                    .thenReturn(Optional.of((QueryBuilder) new TermQueryBuilder("field", values[i])));
+                    .thenReturn(Optional.of(new TermQueryBuilder("field", values[i])));
             when(queryShardContext.toQuery(any(QueryBuilder.class))).thenReturn(parsedQuery);
             DirectoryReader wrappedDirectoryReader = wrapper.wrap(directoryReader);
             IndexSearcher indexSearcher = wrapper.wrap(new IndexSearcher(wrappedDirectoryReader));

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java

@@ -35,33 +35,48 @@ import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.SparseFixedBitSet;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
+import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader;
 import org.elasticsearch.xpack.security.SecurityLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.xpack.security.user.User;
 import org.junit.After;
 import org.junit.Before;
+import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.IdentityHashMap;
+import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.emptyList;
@@ -75,13 +90,18 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
 public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
 
     private ThreadContext threadContext;
     private MapperService mapperService;
+    private ScriptService scriptService;
     private SecurityIndexSearcherWrapper securityIndexSearcherWrapper;
     private ElasticsearchDirectoryReader esIn;
     private SecurityLicenseState licenseState;
@@ -90,6 +110,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
     @Before
     public void before() throws Exception {
         Index index = new Index("_index", "testUUID");
+        scriptService = mock(ScriptService.class);
         indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
         AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(),
                 Collections.emptyMap(), Collections.emptyMap());
@@ -125,7 +146,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
         mapperService.merge("type", new CompressedXContent(mappingSource.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
 
         securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState) {
+                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState, scriptService) {
             @Override
             protected IndicesAccessControl getIndicesAccessControl() {
                 IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true,
@@ -156,14 +177,14 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
     public void testWrapReaderWhenFeatureDisabled() throws Exception {
         when(licenseState.documentAndFieldLevelSecurityEnabled()).thenReturn(false);
         securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState);
+                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState, scriptService);
         DirectoryReader reader = securityIndexSearcherWrapper.wrap(esIn);
         assertThat(reader, sameInstance(esIn));
     }
 
     public void testWrapSearcherWhenFeatureDisabled() throws Exception {
         securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState);
+                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState, scriptService);
         IndexSearcher indexSearcher = new IndexSearcher(esIn);
         IndexSearcher result = securityIndexSearcherWrapper.wrap(indexSearcher);
         assertThat(result, sameInstance(indexSearcher));
@@ -275,7 +296,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
         DirectoryReader directoryReader = DocumentSubsetReader.wrap(esIn, bitsetFilterCache, new MatchAllDocsQuery());
         IndexSearcher indexSearcher = new IndexSearcher(directoryReader);
         securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState);
+                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState, scriptService);
         IndexSearcher result = securityIndexSearcherWrapper.wrap(indexSearcher);
         assertThat(result, not(sameInstance(indexSearcher)));
         assertThat(result.getSimilarity(true), sameInstance(indexSearcher.getSimilarity(true)));
@@ -284,7 +305,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
 
     public void testIntersectScorerAndRoleBits() throws Exception {
         securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState);
+                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState, scriptService);
         final Directory directory = newDirectory();
         IndexWriter iw = new IndexWriter(
                 directory,
@@ -373,7 +394,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
 
     private void assertResolvedFields(String expression, String... expectedFields) {
         securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState) {
+                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState, scriptService) {
             @Override
             protected IndicesAccessControl getIndicesAccessControl() {
                 IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true,
@@ -407,6 +428,60 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
         doTestIndexSearcherWrapper(false, true);
     }
 
+    public void testTemplating() throws Exception {
+        User user = new User("_username", new String[]{"role1", "role2"}, "_full_name", "_email",
+                Collections.singletonMap("key", "value"));
+        securityIndexSearcherWrapper =
+                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState, scriptService) {
+
+                    @Override
+                    protected User getUser() {
+                        return user;
+                    }
+                };
+
+        ExecutableScript executableScript = mock(ExecutableScript.class);
+        when(scriptService.executable(any(Script.class), eq(ScriptContext.Standard.SEARCH), eq(Collections.emptyMap())))
+                .thenReturn(executableScript);
+
+        XContentBuilder builder = jsonBuilder();
+        String query = new TermQueryBuilder("field", "{{_user.username}}").toXContent(builder, ToXContent.EMPTY_PARAMS).string();
+        Script script = new Script(query, ScriptService.ScriptType.INLINE, null, Collections.singletonMap("custom", "value"));
+        builder = jsonBuilder().startObject().field("template");
+        script.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        BytesReference querySource = builder.endObject().bytes();
+
+        securityIndexSearcherWrapper.evaluateTemplate(querySource);
+        ArgumentCaptor<Script> argument = ArgumentCaptor.forClass(Script.class);
+        verify(scriptService).executable(argument.capture(), eq(ScriptContext.Standard.SEARCH), eq(Collections.emptyMap()));
+        Script usedScript = argument.getValue();
+        assertThat(usedScript.getScript(), equalTo(script.getScript()));
+        assertThat(usedScript.getType(), equalTo(script.getType()));
+        assertThat(usedScript.getLang(), equalTo("mustache"));
+        assertThat(usedScript.getContentType(), equalTo(script.getContentType()));
+        assertThat(usedScript.getParams().size(), equalTo(2));
+        assertThat(usedScript.getParams().get("custom"), equalTo("value"));
+
+        Map<String, Object> userModel = new HashMap<>();
+        userModel.put("username", user.principal());
+        userModel.put("full_name", user.fullName());
+        userModel.put("email", user.email());
+        userModel.put("roles", Arrays.asList(user.roles()));
+        userModel.put("metadata", user.metadata());
+        assertThat(usedScript.getParams().get("_user"), equalTo(userModel));
+
+    }
+
+    public void testSkipTemplating() throws Exception {
+        securityIndexSearcherWrapper =
+                new SecurityIndexSearcherWrapper(indexSettings, null, mapperService, null, threadContext, licenseState, scriptService);
+        XContentBuilder builder = jsonBuilder();
+        BytesReference querySource =  new TermQueryBuilder("field", "value").toXContent(builder, ToXContent.EMPTY_PARAMS).bytes();
+        BytesReference result = securityIndexSearcherWrapper.evaluateTemplate(querySource);
+        assertThat(result, sameInstance(querySource));
+        verifyZeroInteractions(scriptService);
+    }
+
     static class CreateScorerOnceWeight extends Weight {
 
         private final Weight weight;

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/ExecutableHttpInput.java

@@ -53,10 +53,14 @@ public class ExecutableHttpInput extends ExecutableInput<HttpInput, HttpInput.Re
     HttpInput.Result doExecute(WatchExecutionContext ctx, HttpRequest request) throws Exception {
         HttpResponse response = client.execute(request);
         Map<String, List<String>> headers = response.headers();
+        Map<String, Object> payloadMap = new HashMap<>();
+        payloadMap.put("_status_code", response.status());
+        if (headers.isEmpty() == false) {
+            payloadMap.put("_headers", headers);
+        }
 
         if (!response.hasContent()) {
-            Payload payload = headers.size() > 0 ? new Payload.Simple("_headers", headers) : Payload.EMPTY;
+            return new HttpInput.Result(request, response.status(), new Payload.Simple(payloadMap));
-            return new HttpInput.Result(request, -1, payload);
         }
 
         final XContentType contentType;
@@ -72,7 +76,6 @@ public class ExecutableHttpInput extends ExecutableInput<HttpInput, HttpInput.Re
             }
         }
 
-        final Map<String, Object> payloadMap = new HashMap<>();
         if (contentType != null) {
             try (XContentParser parser = contentType.xContent().createParser(response.body())) {
                 if (input.getExtractKeys() != null) {
@@ -88,9 +91,6 @@ public class ExecutableHttpInput extends ExecutableInput<HttpInput, HttpInput.Re
             payloadMap.put("_value", response.body().utf8ToString());
         }
 
-        if (headers.size() > 0) {
-            payloadMap.put("_headers", headers);
-        }
         return new HttpInput.Result(request, response.status(), new Payload.Simple(payloadMap));
     }
 }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/http/HttpInput.java

@@ -143,7 +143,7 @@ public class HttpInput implements Input {
     public static class Result extends Input.Result {
 
         @Nullable private final HttpRequest request;
-        private final int statusCode;
+        final int statusCode;
 
         public Result(HttpRequest request, int statusCode, Payload payload) {
             super(TYPE, payload);

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchStatus.java

@@ -16,10 +16,10 @@ import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.support.clock.SystemClock;
 import org.elasticsearch.xpack.watcher.actions.Action;
 import org.elasticsearch.xpack.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.watcher.actions.throttler.AckThrottler;
-import org.elasticsearch.xpack.support.clock.SystemClock;
 import org.elasticsearch.xpack.watcher.support.xcontent.WatcherXContentParser;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -156,7 +156,7 @@ public class WatchStatus implements ToXContent, Streamable {
             dirty = true;
         } else {
             for (ActionStatus status : actions.values()) {
-                status.resetAckStatus(timestamp);
+                dirty |= status.resetAckStatus(timestamp);
             }
         }
     }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java

@@ -289,6 +289,21 @@ public class HttpInputTests extends ESTestCase {
         assertThat(result.payload().data(), not(hasKey("foo")));
     }
 
+    public void testThatStatusCodeIsSetInResultAndPayload() throws Exception {
+        HttpResponse response = new HttpResponse(200);
+        when(httpClient.execute(any(HttpRequest.class))).thenReturn(response);
+
+        HttpRequestTemplate.Builder request = HttpRequestTemplate.builder("localhost", 8080);
+        HttpInput httpInput = InputBuilders.httpInput(request.build()).build();
+        ExecutableHttpInput input = new ExecutableHttpInput(httpInput, logger, httpClient, templateEngine);
+
+        WatchExecutionContext ctx = createWatchExecutionContext();
+        HttpInput.Result result = input.execute(ctx, new Payload.Simple());
+        assertThat(result.statusCode, is(200));
+        assertThat(result.payload().data(), hasKey("_status_code"));
+        assertThat(result.payload().data().get("_status_code"), is(200));
+    }
+
     private WatchExecutionContext createWatchExecutionContext() {
         Watch watch = new Watch("test-watch",
                 new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))),

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStatusTests.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher.watch;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.watcher.actions.ActionStatus;
+import org.elasticsearch.xpack.watcher.actions.ActionStatus.AckStatus.State;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.core.Is.is;
+import static org.joda.time.DateTime.now;
+
+public class WatchStatusTests extends ESTestCase {
+
+    public void testThatWatchStatusDirtyOnConditionCheck() throws Exception {
+        // no actions, met condition
+        WatchStatus status = new WatchStatus(now(), new HashMap<>());
+        status.onCheck(true, now());
+        assertThat(status.dirty(), is(true));
+
+        // no actions, unmet condition
+        status = new WatchStatus(now(), new HashMap<>());
+        status.onCheck(false, now());
+        assertThat(status.dirty(), is(false));
+
+        // actions, no action with reset ack status, unmet condition
+        Map<String, ActionStatus > actions = new HashMap<>();
+        actions.put(randomAsciiOfLength(10), new ActionStatus(now()));
+        status = new WatchStatus(now(), actions);
+        status.onCheck(false, now());
+        assertThat(status.dirty(), is(false));
+
+        // actions, one action with state other than AWAITS_SUCCESSFUL_EXECUTION, unmet condition
+        actions.clear();
+        ActionStatus.AckStatus ackStatus = new ActionStatus.AckStatus(now(), randomFrom(State.ACKED, State.ACKABLE));
+        actions.put(randomAsciiOfLength(10), new ActionStatus(ackStatus, null, null, null));
+        actions.put(randomAsciiOfLength(11), new ActionStatus(now()));
+        status = new WatchStatus(now(), actions);
+        status.onCheck(false, now());
+        assertThat(status.dirty(), is(true));
+    }
+
+}