Merge branch 'master' into reporting-controls

Original commit: elastic/x-pack-elasticsearch@4f0e7501ce

elasticsearch/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java

@@ -6,9 +6,11 @@
 package org.elasticsearch.xpack.security.audit;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.settings.Settings;
@@ -19,7 +21,6 @@ import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Collection;
@@ -35,12 +36,12 @@ public class IndexAuditIT extends ESIntegTestCase {
     private static final String USER = "test_user";
     private static final String PASS = "changeme";
 
-    public void testIndexAuditTrailWorking() throws Exception {
+    public void testShieldIndexAuditTrailWorking() throws Exception {
-        HttpResponse response = httpClient().path("/")
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
-                .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(USER, new SecuredString(PASS.toCharArray())))
+                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                .execute();
+                        UsernamePasswordToken.basicAuthHeaderValue(USER, new SecuredString(PASS.toCharArray()))))) {
-        assertThat(response.getStatusCode(), is(200));
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
-
+        }
         final AtomicReference<ClusterState> lastClusterState = new AtomicReference<>();
         final AtomicBoolean indexExists = new AtomicBoolean(false);
         boolean found = awaitBusy(() -> {

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyScriptConditionIT.java

@@ -66,13 +66,10 @@ public class GroovyScriptConditionIT extends AbstractWatcherIntegrationTestCase
     }
 
     public void testGroovyClosureWithAggregations() throws Exception {
-        client().admin().indices().prepareCreate(".monitoring")
-                .addMapping("cluster_stats", "_timestamp", "enabled=true")
-                .get();
-
         for (int seconds = 0; seconds < 60; seconds += 5) {
-            client().prepareIndex(".monitoring", "cluster_stats").setTimestamp("2005-01-01T00:00:" +
+            String timestamp = "2005-01-01T00:00:" + String.format(Locale.ROOT, "%02d", seconds);
-                    String.format(Locale.ROOT, "%02d", seconds)).setSource("status", randomFrom("green", "yellow")).get();
+            client().prepareIndex(".monitoring", "cluster_stats")
+                    .setSource("status", randomFrom("green", "yellow"), "@timestamp", timestamp).get();
         }
 
         refresh();
@@ -80,7 +77,7 @@ public class GroovyScriptConditionIT extends AbstractWatcherIntegrationTestCase
         SearchRequestBuilder builder = client().prepareSearch(".monitoring")
                 .addAggregation(
                         AggregationBuilders
-                                .dateHistogram("minutes").field("_timestamp").interval(TimeUnit.MILLISECONDS.convert(5, TimeUnit.SECONDS))
+                                .dateHistogram("minutes").field("@timestamp").interval(TimeUnit.MILLISECONDS.convert(5, TimeUnit.SECONDS))
                                 .order(Histogram.Order.COUNT_DESC)
                                 .subAggregation(AggregationBuilders.terms("status").field("status.keyword").size(3)));
         SearchResponse unmetResponse = builder.get();
@@ -101,8 +98,8 @@ public class GroovyScriptConditionIT extends AbstractWatcherIntegrationTestCase
         assertFalse(condition.execute(unmetContext).met());
 
         for (int seconds = 0; seconds < 60; seconds += 5) {
-            client().prepareIndex(".monitoring", "cluster_stats").setTimestamp("2005-01-01T00:01:" +
+            String timestamp = "2005-01-01T00:01:" + String.format(Locale.ROOT, "%02d", seconds);
-                    String.format(Locale.ROOT, "%02d", seconds)).setSource("status", randomFrom("red")).get();
+            client().prepareIndex(".monitoring", "cluster_stats").setSource("status", randomFrom("red"), "@timestamp", timestamp).get();
         }
 
         refresh();

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchIT.java

@@ -20,11 +20,11 @@ import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.common.ScriptServiceProxy;
 import org.elasticsearch.xpack.watcher.condition.script.ExecutableScriptCondition;
 import org.elasticsearch.xpack.watcher.condition.script.ScriptCondition;
 import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.watcher.support.Script;
-import org.elasticsearch.xpack.common.ScriptServiceProxy;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.watch.Payload;
 import org.junit.After;
@@ -61,18 +61,14 @@ public class ScriptConditionSearchIT extends AbstractWatcherIntegrationTestCase
     }
 
     public void testExecuteWithAggs() throws Exception {
-        client().admin().indices().prepareCreate("my-index")
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:00").get();
-                .addMapping("my-type", "_timestamp", "enabled=true")
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:10").get();
-                .get();
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:20").get();
-
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:30").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:00").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:10").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:20").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:30").setSource("{}").get();
         refresh();
 
         SearchResponse response = client().prepareSearch("my-index")
-                .addAggregation(AggregationBuilders.dateHistogram("rate").field("_timestamp")
+                .addAggregation(AggregationBuilders.dateHistogram("rate").field("@timestamp")
                         .dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC))
                 .get();
 
@@ -83,10 +79,10 @@ public class ScriptConditionSearchIT extends AbstractWatcherIntegrationTestCase
         WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response));
         assertFalse(condition.execute(ctx).met());
 
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:40").setSource("{}").get();
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:40").get();
         refresh();
 
-        response = client().prepareSearch("my-index").addAggregation(AggregationBuilders.dateHistogram("rate").field("_timestamp")
+        response = client().prepareSearch("my-index").addAggregation(AggregationBuilders.dateHistogram("rate").field("@timestamp")
                 .dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC))
                 .get();
 

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java

@@ -12,31 +12,40 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.Template;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.mustache.MustachePlugin;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.xpack.common.ScriptServiceProxy;
 import org.elasticsearch.xpack.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.actions.ActionWrapper;
 import org.elasticsearch.xpack.watcher.actions.ExecutableActions;
 import org.elasticsearch.xpack.watcher.condition.always.ExecutableAlwaysCondition;
 import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext;
 import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
+import org.elasticsearch.xpack.watcher.input.Input;
 import org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput;
 import org.elasticsearch.xpack.watcher.input.search.SearchInput;
 import org.elasticsearch.xpack.watcher.input.search.SearchInputFactory;
 import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
+import org.elasticsearch.xpack.watcher.support.Script;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
+import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent;
@@ -67,7 +76,6 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.joda.time.DateTimeZone.UTC;
 
 /**
@@ -81,10 +89,11 @@ public class SearchInputIT extends ESIntegTestCase {
         Collection<Class<? extends Plugin>> types = new ArrayList<>();
         types.addAll(super.nodePlugins());
         types.add(MustachePlugin.class);
+        types.add(CustomScriptContextPlugin.class);
         return types;
     }
 
-    private final static String TEMPLATE_QUERY = "{\"query\":{\"filtered\":{\"query\":{\"match\":{\"event_type\":{\"query\":\"a\"," +
+    private final static String TEMPLATE_QUERY = "{\"query\":{\"bool\":{\"must\":{\"match\":{\"event_type\":{\"query\":\"a\"," +
             "\"type\":\"boolean\"}}},\"filter\":{\"range\":{\"_timestamp\":" +
             "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," +
             "\"include_lower\":true,\"include_upper\":true}}}}}}";
@@ -100,7 +109,6 @@ public class SearchInputIT extends ESIntegTestCase {
             throw new RuntimeException("failed to create config dir");
 
         }
-        String path = "/org/elasticsearch/xpack/watcher/input/search/config/scripts/test_disk_template.mustache";
         try (InputStream stream  = SearchInputIT.class.getResourceAsStream("/org/elasticsearch/xpack/watcher/input/search/config/scripts" +
                 "/test_disk_template.mustache");
              OutputStream out = Files.newOutputStream(scriptPath.resolve("test_disk_template.mustache"))) {
@@ -126,14 +134,15 @@ public class SearchInputIT extends ESIntegTestCase {
         SearchSourceBuilder searchSourceBuilder = searchSource().query(
                 boolQuery().must(matchQuery("event_type", "a")).must(rangeQuery("_timestamp")
                         .from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}")));
-        SearchRequest request = client()
+        SearchRequest searchRequest = client()
                 .prepareSearch()
                 .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE)
                 .request()
                 .source(searchSourceBuilder);
 
+        WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(searchRequest);
         ExecutableSearchInput searchInput = new ExecutableSearchInput(new SearchInput(request, null, null, null), logger,
-                WatcherClientProxy.of(client()), null);
+                WatcherClientProxy.of(client()), watcherSearchTemplateService(), null);
         WatchExecutionContext ctx = new TriggeredExecutionContext(
                 new Watch("test-watch",
                         new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))),
@@ -149,21 +158,21 @@ public class SearchInputIT extends ESIntegTestCase {
                 timeValueSeconds(5));
         SearchInput.Result result = searchInput.execute(ctx, new Payload.Simple());
 
-        assertThat((Integer) XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0));
+        assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0));
         assertNotNull(result.executedRequest());
-        assertEquals(result.executedRequest().searchType(), request.searchType());
+        assertThat(result.status(), is(Input.Result.Status.SUCCESS));
-        assertArrayEquals(result.executedRequest().indices(), request.indices());
+        assertEquals(result.executedRequest().searchType(), request.getRequest().searchType());
-        assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions());
+        assertArrayEquals(result.executedRequest().indices(), request.getRequest().indices());
+        assertEquals(result.executedRequest().indicesOptions(), request.getRequest().indicesOptions());
+
+        XContentSource source = toXContentSource(result);
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:00:00.000Z||-30s"));
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:00:00.000Z"));
     }
 
     public void testSearchInlineTemplate() throws Exception {
         WatchExecutionContext ctx = createContext();
 
-        final String expectedTemplateString = "{\"query\":{\"filtered\":{\"query\":{\"match\":{\"event_type\":{\"query\":\"a\","
-                + "\"type\":\"boolean\"}}},\"filter\":{\"range\":{\"_timestamp\":"
-                + "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\","
-                + "\"include_lower\":true,\"include_upper\":true}}}}}}";
-
         Map<String, Object> triggerParams = new HashMap<String, Object>();
         triggerParams.put("triggered_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC()));
         triggerParams.put("scheduled_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC()));
@@ -178,17 +187,28 @@ public class SearchInputIT extends ESIntegTestCase {
         Map<String, Object> expectedParams = new HashMap<String, Object>();
         expectedParams.put("seconds_param", "30s");
         expectedParams.put("ctx", ctxParams);
-        Template expectedTemplate = new Template(expectedTemplateString, ScriptType.INLINE, null, XContentType.JSON, expectedParams);
         Map<String, Object> params = new HashMap<>();
         params.put("seconds_param", "30s");
 
-        Template template = new Template(TEMPLATE_QUERY, ScriptType.INLINE, null, XContentType.JSON, params);
+        Script template = Script.inline(TEMPLATE_QUERY).lang("mustache").params(params).build();
 
-        SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE)
+        SearchRequest request = client().prepareSearch()
-                .setIndices("test-search-index").setTemplate(template).request();
+                .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE)
+                .setIndices("test-search-index").request();
 
-        SearchInput.Result executedResult = executeSearchInput(request, ctx);
+        SearchInput.Result executedResult = executeSearchInput(request, template, ctx);
-        assertThat(executedResult.executedRequest().template(), equalTo(expectedTemplate));
+
+        assertNotNull(executedResult.executedRequest());
+        assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS));
+        if (getNumShards("test-search-index").numPrimaries > 1) {
+            assertEquals(executedResult.executedRequest().searchType(), request.searchType());
+        }
+        assertArrayEquals(executedResult.executedRequest().indices(), request.indices());
+        assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions());
+
+        XContentSource source = toXContentSource(executedResult);
+        assertThat(source.getValue("query.bool.filter.0.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s"));
+        assertThat(source.getValue("query.bool.filter.0.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z"));
     }
 
     public void testSearchIndexedTemplate() throws Exception {
@@ -204,17 +224,26 @@ public class SearchInputIT extends ESIntegTestCase {
         Map<String, Object> params = new HashMap<>();
         params.put("seconds_param", "30s");
 
-        Template template = new Template("test-template", ScriptType.STORED, null, null, params);
+        Script template = Script.indexed("test-template").lang("mustache").params(params).build();
 
         jsonBuilder().value(TextTemplate.indexed("test-template").params(params).build()).bytes();
         SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE)
-                .setIndices("test-search-index").setTemplate(template).request();
+                .setIndices("test-search-index").request();
+
+        SearchInput.Result executedResult = executeSearchInput(request, template, ctx);
+
+        assertNotNull(executedResult.executedRequest());
+        assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS));
+        if (getNumShards("test-search-index").numPrimaries > 1) {
+            assertEquals(executedResult.executedRequest().searchType(), request.searchType());
+        }
+        assertArrayEquals(executedResult.executedRequest().indices(), request.indices());
+        assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions());
+
+        XContentSource source = toXContentSource(executedResult);
+        assertThat(source.getValue("query.bool.filter.0.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s"));
+        assertThat(source.getValue("query.bool.filter.0.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z"));
 
-        SearchInput.Result executedResult = executeSearchInput(request, ctx);
-        Template resultTemplate = executedResult.executedRequest().template();
-        assertThat(resultTemplate, notNullValue());
-        assertThat(resultTemplate.getScript(), equalTo("test-template"));
-        assertThat(resultTemplate.getType(), equalTo(ScriptType.STORED));
     }
 
     public void testSearchOnDiskTemplate() throws Exception {
@@ -223,15 +252,16 @@ public class SearchInputIT extends ESIntegTestCase {
         Map<String, Object> params = new HashMap<>();
         params.put("seconds_param", "30s");
 
-        Template template = new Template("test_disk_template", ScriptType.FILE, null, null, params);
+        Script template = Script.file("test_disk_template").lang("mustache").params(params).build();
         SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE)
-                .setIndices("test-search-index").setTemplate(template).request();
+                .setIndices("test-search-index").request();
 
-        SearchInput.Result executedResult = executeSearchInput(request, ctx);
+        SearchInput.Result executedResult = executeSearchInput(request, template, ctx);
-        Template resultTemplate = executedResult.executedRequest().template();
+
-        assertThat(resultTemplate, notNullValue());
+        assertNotNull(executedResult.executedRequest());
-        assertThat(resultTemplate.getScript(), equalTo("test_disk_template"));
+        assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS));
-        assertThat(resultTemplate.getType(), equalTo(ScriptType.FILE));
+        assertArrayEquals(executedResult.executedRequest().indices(), request.indices());
+        assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions());
     }
 
     public void testDifferentSearchType() throws Exception {
@@ -241,14 +271,16 @@ public class SearchInputIT extends ESIntegTestCase {
         );
         SearchType searchType = getRandomSupportedSearchType();
 
-        SearchRequest request = client()
+        SearchRequest searchRequest = client()
                 .prepareSearch()
                 .setSearchType(searchType)
                 .request()
                 .source(searchSourceBuilder);
 
+        WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(searchRequest);
+
         ExecutableSearchInput searchInput = new ExecutableSearchInput(new SearchInput(request, null, null, null), logger,
-                WatcherClientProxy.of(client()), null);
+                WatcherClientProxy.of(client()), watcherSearchTemplateService(), null);
         WatchExecutionContext ctx = new TriggeredExecutionContext(
                 new Watch("test-watch",
                         new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))),
@@ -264,15 +296,20 @@ public class SearchInputIT extends ESIntegTestCase {
                 timeValueSeconds(5));
         SearchInput.Result result = searchInput.execute(ctx, new Payload.Simple());
 
-        assertThat((Integer) XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0));
+        assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0));
         assertNotNull(result.executedRequest());
+        assertThat(result.status(), is(Input.Result.Status.SUCCESS));
         assertEquals(result.executedRequest().searchType(), searchType);
-        assertArrayEquals(result.executedRequest().indices(), request.indices());
+        assertArrayEquals(result.executedRequest().indices(), searchRequest.indices());
-        assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions());
+        assertEquals(result.executedRequest().indicesOptions(), searchRequest.indicesOptions());
+
+        XContentSource source = toXContentSource(result);
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:00:00.000Z||-30s"));
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:00:00.000Z"));
     }
 
     public void testParserValid() throws Exception {
-        SearchRequest request = client().prepareSearch()
+        SearchRequest searchRequest = client().prepareSearch()
                 .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE)
                 .request()
                 .source(searchSource()
@@ -280,13 +317,14 @@ public class SearchInputIT extends ESIntegTestCase {
                                 .from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}"))));
 
         TimeValue timeout = randomBoolean() ? TimeValue.timeValueSeconds(randomInt(10)) : null;
-        XContentBuilder builder = jsonBuilder().value(new SearchInput(request, null, timeout, null));
+        XContentBuilder builder = jsonBuilder().value(
+                new SearchInput(new WatcherSearchTemplateRequest(searchRequest), null, timeout, null));
         XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes());
         parser.nextToken();
 
         IndicesQueriesRegistry indicesQueryRegistry = internalCluster().getInstance(IndicesQueriesRegistry.class);
         SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, WatcherClientProxy.of(client()), indicesQueryRegistry,
-                                                            null, null);
+                                                            null, null, scriptService());
 
         SearchInput searchInput = factory.parseInput("_id", parser);
         assertEquals(SearchInput.TYPE, searchInput.type());
@@ -309,15 +347,47 @@ public class SearchInputIT extends ESIntegTestCase {
                 timeValueSeconds(5));
     }
 
-    private SearchInput.Result executeSearchInput(SearchRequest request, WatchExecutionContext ctx) throws IOException {
+    private SearchInput.Result executeSearchInput(SearchRequest request, Script template, WatchExecutionContext ctx) throws IOException {
         createIndex("test-search-index");
         ensureGreen("test-search-index");
-        SearchInput.Builder siBuilder = SearchInput.builder(request);
+        SearchInput.Builder siBuilder = SearchInput.builder(new WatcherSearchTemplateRequest(request, template));
 
         SearchInput si = siBuilder.build();
 
-        ExecutableSearchInput searchInput = new ExecutableSearchInput(si, logger, WatcherClientProxy.of(client()), null);
+        ExecutableSearchInput searchInput = new ExecutableSearchInput(si, logger, WatcherClientProxy.of(client()),
+                watcherSearchTemplateService(), null);
         return searchInput.execute(ctx, new Payload.Simple());
     }
 
+    protected WatcherSearchTemplateService watcherSearchTemplateService() {
+        String master = internalCluster().getMasterName();
+        return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(),
+                ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master), internalCluster().clusterService(master)),
+                internalCluster().getInstance(IndicesQueriesRegistry.class, master),
+                internalCluster().getInstance(AggregatorParsers.class, master),
+                internalCluster().getInstance(Suggesters.class, master)
+        );
+    }
+
+    protected ScriptServiceProxy scriptService() {
+        return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class), internalCluster().clusterService());
+    }
+
+    private XContentSource toXContentSource(SearchInput.Result result) throws IOException {
+        try (XContentBuilder builder = jsonBuilder()) {
+            result.executedRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS);
+            return new XContentSource(builder);
+        }
+    }
+
+    /**
+     * Custom plugin that registers XPack script context.
+     */
+    public static class CustomScriptContextPlugin extends Plugin implements ScriptPlugin {
+
+        @Override
+        public ScriptContext.Plugin getCustomScriptContexts() {
+            return ScriptServiceProxy.INSTANCE;
+        }
+    }
 }

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java

@@ -11,25 +11,28 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.script.Template;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.mustache.MustachePlugin;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.xpack.common.ScriptServiceProxy;
 import org.elasticsearch.xpack.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.actions.ExecutableActions;
 import org.elasticsearch.xpack.watcher.condition.always.ExecutableAlwaysCondition;
@@ -37,7 +40,11 @@ import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext;
 import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
+import org.elasticsearch.xpack.watcher.support.Script;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
+import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.watcher.transform.Transform;
 import org.elasticsearch.xpack.watcher.transform.TransformBuilders;
 import org.elasticsearch.xpack.watcher.transform.search.ExecutableSearchTransform;
@@ -100,6 +107,7 @@ public class SearchTransformIT extends ESIntegTestCase {
         Collection<Class<? extends Plugin>> types = new ArrayList<>();
         types.addAll(super.nodePlugins());
         types.add(MustachePlugin.class);
+        types.add(CustomScriptContextPlugin.class);
         return types;
     }
 
@@ -154,7 +162,8 @@ public class SearchTransformIT extends ESIntegTestCase {
 
         SearchRequest request = Requests.searchRequest("idx").source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()));
         SearchTransform searchTransform = TransformBuilders.searchTransform(request).build();
-        ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), null);
+        ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()),
+                watcherSearchTemplateService(), null);
 
         WatchExecutionContext ctx = mockExecutionContext("_name", EMPTY_PAYLOAD);
 
@@ -188,7 +197,8 @@ public class SearchTransformIT extends ESIntegTestCase {
                 new SearchSourceBuilder().query(QueryBuilders.wrapperQuery(jsonBuilder().startObject()
                         .startObject("_unknown_query_").endObject().endObject().bytes())));
         SearchTransform searchTransform = TransformBuilders.searchTransform(request).build();
-        ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), null);
+        ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()),
+                watcherSearchTemplateService(), null);
 
         WatchExecutionContext ctx = mockExecutionContext("_name", EMPTY_PAYLOAD);
 
@@ -200,23 +210,27 @@ public class SearchTransformIT extends ESIntegTestCase {
         assertThat(result.reason(), containsString("no [query] registered for [_unknown_query_]"));
 
         // extract the base64 encoded query from the template script, path is: query -> wrapper -> query
-        String jsonQuery = result.executedRequest().template().getScript();
+        try (XContentBuilder builder = jsonBuilder()) {
-        Map<String, Object> map = XContentFactory.xContent(jsonQuery).createParser(jsonQuery).map();
+            result.executedRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS);
 
-        assertThat(map, hasKey("query"));
+            String jsonQuery = builder.string();
-        assertThat(map.get("query"), instanceOf(Map.class));
+            Map<String, Object> map = XContentFactory.xContent(jsonQuery).createParser(jsonQuery).map();
 
-        map = (Map<String, Object>) map.get("query");
+            assertThat(map, hasKey("query"));
-        assertThat(map, hasKey("wrapper"));
+            assertThat(map.get("query"), instanceOf(Map.class));
-        assertThat(map.get("wrapper"), instanceOf(Map.class));
 
-        map = (Map<String, Object>) map.get("wrapper");
+            map = (Map<String, Object>) map.get("query");
-        assertThat(map, hasKey("query"));
+            assertThat(map, hasKey("wrapper"));
-        assertThat(map.get("query"), instanceOf(String.class));
+            assertThat(map.get("wrapper"), instanceOf(Map.class));
 
-        String queryAsBase64 = (String) map.get("query");
+            map = (Map<String, Object>) map.get("wrapper");
-        String decodedQuery = new String(Base64.getDecoder().decode(queryAsBase64), StandardCharsets.UTF_8);
+            assertThat(map, hasKey("query"));
-        assertThat(decodedQuery, containsString("_unknown_query_"));
+            assertThat(map.get("query"), instanceOf(String.class));
+
+            String queryAsBase64 = (String) map.get("query");
+            String decodedQuery = new String(Base64.getDecoder().decode(queryAsBase64), StandardCharsets.UTF_8);
+            assertThat(decodedQuery, containsString("_unknown_query_"));
+        }
     }
 
     public void testExecuteMustacheTemplate() throws Exception {
@@ -252,7 +266,8 @@ public class SearchTransformIT extends ESIntegTestCase {
                 .must(termQuery("value", "{{ctx.payload.value}}"))));
 
         SearchTransform searchTransform = TransformBuilders.searchTransform(request).build();
-        ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), null);
+        ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()),
+                watcherSearchTemplateService(), null);
 
         ScheduleTriggerEvent event = new ScheduleTriggerEvent("_name", parseDate("2015-01-04T00:00:00", UTC),
                 parseDate("2015-01-01T00:00:00", UTC));
@@ -319,24 +334,24 @@ public class SearchTransformIT extends ESIntegTestCase {
 
         IndicesQueriesRegistry indicesQueryRegistry = internalCluster().getInstance(IndicesQueriesRegistry.class);
         SearchTransformFactory transformFactory = new SearchTransformFactory(Settings.EMPTY, WatcherClientProxy.of(client()),
-                                                                             indicesQueryRegistry, null, null);
+                                                                             indicesQueryRegistry, null, null, scriptService());
         ExecutableSearchTransform executable = transformFactory.parseExecutable("_id", parser);
 
         assertThat(executable, notNullValue());
         assertThat(executable.type(), is(SearchTransform.TYPE));
         assertThat(executable.transform().getRequest(), notNullValue());
         if (indices != null) {
-            assertThat(executable.transform().getRequest().indices(), arrayContainingInAnyOrder(indices));
+            assertThat(executable.transform().getRequest().getRequest().indices(), arrayContainingInAnyOrder(indices));
         }
         if (searchType != null) {
-            assertThat(executable.transform().getRequest().searchType(), is(searchType));
+            assertThat(executable.transform().getRequest().getRequest().searchType(), is(searchType));
         }
         if (templateName != null) {
-            assertThat(executable.transform().getRequest().template(),
+            assertThat(executable.transform().getRequest().getTemplate(),
-                    equalTo(new Template("template1", ScriptType.FILE, null, null, null)));
+                    equalTo(Script.file("template1").build()));
         }
         SearchSourceBuilder source = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery());
-        assertThat(executable.transform().getRequest().source(), equalTo(source));
+        assertThat(executable.transform().getRequest().getRequest().source(), equalTo(source));
         assertThat(executable.transform().getTimeout(), equalTo(readTimeout));
     }
 
@@ -348,11 +363,6 @@ public class SearchTransformIT extends ESIntegTestCase {
                 "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," +
                 "\"include_lower\":true,\"include_upper\":true}}}]}}}";
 
-        final String expectedTemplateString = "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"event_type\":{\"query\":\"a\","
-                + "\"type\":\"boolean\"}}},{\"range\":{\"_timestamp\":"
-                + "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\","
-                + "\"include_lower\":true,\"include_upper\":true}}}]}}}";
-
         Map<String, Object> triggerParams = new HashMap<String, Object>();
         triggerParams.put("triggered_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC()));
         triggerParams.put("scheduled_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC()));
@@ -367,18 +377,24 @@ public class SearchTransformIT extends ESIntegTestCase {
         Map<String, Object> expectedParams = new HashMap<String, Object>();
         expectedParams.put("seconds_param", "30s");
         expectedParams.put("ctx", ctxParams);
-        Template expectedTemplate = new Template(expectedTemplateString, ScriptType.INLINE, null, XContentType.JSON, expectedParams);
 
         Map<String, Object> params = new HashMap<>();
         params.put("seconds_param", "30s");
 
-        Template template = new Template(templateQuery, ScriptType.INLINE, null, XContentType.JSON, params);
+        Script template = Script.inline(templateQuery).lang("mustache").params(params).build();
         SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE)
-                .setIndices("test-search-index").setTemplate(template).request();
+                .setIndices("test-search-index").request();
 
-        SearchTransform.Result executedResult = executeSearchTransform(request, ctx);
+        SearchTransform.Result executedResult = executeSearchTransform(request, template, ctx);
 
-        assertThat(executedResult.executedRequest().template(), equalTo(expectedTemplate));
+        assertThat(executedResult.status(), is(Transform.Result.Status.SUCCESS));
+        assertEquals(executedResult.executedRequest().searchType(), request.searchType());
+        assertArrayEquals(executedResult.executedRequest().indices(), request.indices());
+        assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions());
+
+        XContentSource source = toXContentSource(executedResult);
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s"));
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z"));
     }
 
     public void testSearchIndexedTemplate() throws Exception {
@@ -399,24 +415,24 @@ public class SearchTransformIT extends ESIntegTestCase {
         Map<String, Object> params = new HashMap<>();
         params.put("seconds_param", "30s");
 
-        BytesReference templateSource = jsonBuilder()
+        Script template = Script.indexed("test-script").lang("mustache").params(params).build();
-                .value(TextTemplate.indexed("test-script").params(params).build())
-                .bytes();
-        Template template = new Template("test-script", ScriptType.STORED, null, null, null);
 
         SearchRequest request = client()
                 .prepareSearch()
                 .setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE)
                 .setIndices("test-search-index")
-                .setTemplate(template)
                 .request();
 
-        SearchTransform.Result result = executeSearchTransform(request, ctx);
+        SearchTransform.Result result = executeSearchTransform(request, template, ctx);
+
         assertNotNull(result.executedRequest());
-        Template resultTemplate = result.executedRequest().template();
+        assertThat(result.status(), is(Transform.Result.Status.SUCCESS));
-        assertThat(resultTemplate, notNullValue());
+        assertArrayEquals(result.executedRequest().indices(), request.indices());
-        assertThat(resultTemplate.getScript(), equalTo("test-script"));
+        assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions());
-        assertThat(resultTemplate.getType(), equalTo(ScriptType.STORED));
+
+        XContentSource source = toXContentSource(result);
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s"));
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z"));
     }
 
     public void testSearchOnDiskTemplate() throws Exception {
@@ -425,16 +441,20 @@ public class SearchTransformIT extends ESIntegTestCase {
         Map<String, Object> params = new HashMap<>();
         params.put("seconds_param", "30s");
 
-        Template template = new Template("test_disk_template", ScriptType.FILE, null, null, null);
+        Script template = Script.file("test_disk_template").lang("mustache").params(params).build();
         SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE)
-                .setIndices("test-search-index").setTemplate(template).request();
+                .setIndices("test-search-index").request();
+
+        SearchTransform.Result result = executeSearchTransform(request, template, ctx);
 
-        SearchTransform.Result result = executeSearchTransform(request, ctx);
         assertNotNull(result.executedRequest());
-        Template resultTemplate = result.executedRequest().template();
+        assertThat(result.status(), is(Transform.Result.Status.SUCCESS));
-        assertThat(resultTemplate, notNullValue());
+        assertArrayEquals(result.executedRequest().indices(), request.indices());
-        assertThat(resultTemplate.getScript(), equalTo("test_disk_template"));
+        assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions());
-        assertThat(resultTemplate.getType(), equalTo(ScriptType.FILE));
+
+        XContentSource source = toXContentSource(result);
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s"));
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z"));
     }
 
     public void testDifferentSearchType() throws Exception {
@@ -453,13 +473,18 @@ public class SearchTransformIT extends ESIntegTestCase {
                 .request()
                 .source(searchSourceBuilder);
 
-        SearchTransform.Result result = executeSearchTransform(request, ctx);
+        SearchTransform.Result result = executeSearchTransform(request, null, ctx);
 
-        assertThat((Integer) XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0));
+        assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0));
         assertThat(result.executedRequest(), notNullValue());
+        assertThat(result.status(), is(Transform.Result.Status.SUCCESS));
         assertThat(result.executedRequest().searchType(), is(searchType));
         assertThat(result.executedRequest().indices(), arrayContainingInAnyOrder(request.indices()));
         assertThat(result.executedRequest().indicesOptions(), equalTo(request.indicesOptions()));
+
+        XContentSource source = toXContentSource(result);
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s"));
+        assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z"));
     }
 
     private WatchExecutionContext createContext() {
@@ -479,17 +504,31 @@ public class SearchTransformIT extends ESIntegTestCase {
                 timeValueSeconds(5));
     }
 
-    private SearchTransform.Result executeSearchTransform(SearchRequest request, WatchExecutionContext ctx) throws IOException {
+    private SearchTransform.Result executeSearchTransform(SearchRequest request, Script template, WatchExecutionContext ctx)
+            throws IOException {
         createIndex("test-search-index");
         ensureGreen("test-search-index");
 
-        SearchTransform searchTransform = TransformBuilders.searchTransform(request).build();
+        SearchTransform searchTransform = TransformBuilders.searchTransform(new WatcherSearchTemplateRequest(request, template)).build();
         ExecutableSearchTransform executableSearchTransform = new ExecutableSearchTransform(searchTransform, logger,
-                WatcherClientProxy.of(client()), null);
+                WatcherClientProxy.of(client()), watcherSearchTemplateService(), null);
 
         return executableSearchTransform.execute(ctx, Payload.Simple.EMPTY);
     }
 
+    protected WatcherSearchTemplateService watcherSearchTemplateService() {
+        String master = internalCluster().getMasterName();
+        return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(),
+                ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master), internalCluster().clusterService(master)),
+                internalCluster().getInstance(IndicesQueriesRegistry.class, master),
+                internalCluster().getInstance(AggregatorParsers.class, master),
+                internalCluster().getInstance(Suggesters.class, master)
+        );
+    }
+
+    protected ScriptServiceProxy scriptService() {
+        return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class), internalCluster().clusterService());
+    }
 
     private static Map<String, Object> doc(String date, String value) {
         Map<String, Object> doc = new HashMap<>();
@@ -498,4 +537,21 @@ public class SearchTransformIT extends ESIntegTestCase {
         return doc;
     }
 
+    private XContentSource toXContentSource(SearchTransform.Result result) throws IOException {
+        try (XContentBuilder builder = jsonBuilder()) {
+            result.executedRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS);
+            return new XContentSource(builder);
+        }
+    }
+
+    /**
+     * Custom plugin that registers XPack script context.
+     */
+    public static class CustomScriptContextPlugin extends Plugin implements ScriptPlugin {
+
+        @Override
+        public ScriptContext.Plugin getCustomScriptContexts() {
+            return ScriptServiceProxy.INSTANCE;
+        }
+    }
 }

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java

@@ -5,27 +5,27 @@
  */
 package org.elasticsearch.messy.tests;
 
+import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.TermsLookup;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
-import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
+import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.script.ScriptService.ScriptType.INLINE;
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
@@ -72,18 +72,6 @@ public class SecurityCachePermissionIT extends SecurityIntegTestCase {
     public void loadData() {
         index("data", "a", "1", "{ \"name\": \"John\", \"token\": \"token1\" }");
         index("tokens", "tokens", "1", "{ \"group\": \"1\", \"tokens\": [\"token1\", \"token2\"] }");
-        client().admin().cluster().preparePutStoredScript().setSource(new BytesArray("{\n" +
-                "\"template\": {\n" +
-                "  \"query\": {\n" +
-                "    \"exists\": {\n" +
-                "      \"field\": \"{{name}}\"\n" +
-                "     }\n" +
-                "   }\n" +
-                " }\n" +
-                "}"))
-                .setScriptLang("mustache")
-                .setId("testTemplate")
-                .execute().actionGet();
         refresh();
     }
 
@@ -96,38 +84,44 @@ public class SecurityCachePermissionIT extends SecurityIntegTestCase {
 
         // Repeat with unauthorized user!!!!
         try {
-            response = client().filterWithHeader(Collections.singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER,
+            response = client().filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER,
                     new SecuredString("changeme".toCharArray()))))
                     .prepareSearch("data").setTypes("a").setQuery(QueryBuilders.constantScoreQuery(
                     QueryBuilders.termsLookupQuery("token", new TermsLookup("tokens", "tokens", "1", "tokens"))))
                     .execute().actionGet();
             fail("search phase exception should have been thrown! response was:\n" + response.toString());
-        } catch (SearchPhaseExecutionException e) {
+        } catch (ElasticsearchSecurityException e) {
             assertThat(e.toString(), containsString("ElasticsearchSecurityException[action"));
             assertThat(e.toString(), containsString("unauthorized"));
         }
     }
 
     public void testThatScriptServiceDoesntLeakData() {
+        String source = "{\n" +
+                "\"template\": {\n" +
+                "  \"query\": {\n" +
+                "    \"exists\": {\n" +
+                "      \"field\": \"{{name}}\"\n" +
+                "     }\n" +
+                "   }\n" +
+                " }\n" +
+                "}";
+
+        //Template template = new Template(source, INLINE, MustacheScriptEngineService.NAME, null, singletonMap("name", "token"));
         SearchResponse response = client().prepareSearch("data").setTypes("a")
-                .setTemplate(new Template("testTemplate", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null,
+                .setQuery(QueryBuilders.templateQuery(source, singletonMap("name", "token")))
-                        Collections.<String, Object>singletonMap("name", "token")))
                 .execute().actionGet();
         assertThat(response.isTimedOut(), is(false));
         assertThat(response.getHits().hits().length, is(1));
 
         // Repeat with unauthorized user!!!!
-        try {
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> client()
-            response = client().filterWithHeader(Collections.singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER,
+                .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER,
-                    new SecuredString("changeme".toCharArray()))))
+                        new SecuredString("changeme".toCharArray()))))
                     .prepareSearch("data").setTypes("a")
-                    .setTemplate(new Template("testTemplate", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null,
+                    .setQuery(QueryBuilders.templateQuery(source, singletonMap("name", "token")))
-                            Collections.<String, Object>singletonMap("name", "token")))
+                    .execute().actionGet());
-                    .execute().actionGet();
+        assertThat(e.toString(), containsString("ElasticsearchSecurityException[action"));
-            fail("search phase exception should have been thrown! response was:\n" + response.toString());
+        assertThat(e.toString(), containsString("unauthorized"));
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.toString(), containsString("ElasticsearchSecurityException[action"));
-            assertThat(e.toString(), containsString("unauthorized"));
-        }
     }
 }

elasticsearch/qa/security-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java

@@ -5,9 +5,12 @@
  */
 package org.elasticsearch.example.realm;
 
+import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
@@ -16,7 +19,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Collection;
@@ -29,6 +31,7 @@ import static org.hamcrest.Matchers.is;
  * Integration test to test authentication with the custom realm
  */
 public class CustomRealmIT extends ESIntegTestCase {
+
     @Override
     protected Settings externalClusterClientSettings() {
         return Settings.builder()
@@ -43,18 +46,23 @@ public class CustomRealmIT extends ESIntegTestCase {
     }
 
     public void testHttpConnectionWithNoAuthentication() throws Exception {
-        HttpResponse response = httpClient().path("/").execute();
+        try {
-        assertThat(response.getStatusCode(), is(401));
+            getRestClient().performRequest("GET", "/", Collections.emptyMap(), null);
-        String value = response.getHeaders().get("WWW-Authenticate");
+            fail("request should have failed");
-        assertThat(value, is("custom-challenge"));
+        } catch(ResponseException e) {
+            Response response = e.getResponse();
+            assertThat(response.getStatusLine().getStatusCode(), is(401));
+            String value = response.getHeader("WWW-Authenticate");
+            assertThat(value, is("custom-challenge"));
+        }
     }
 
     public void testHttpAuthentication() throws Exception {
-        HttpResponse response = httpClient().path("/")
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
-                .addHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER)
+                new BasicHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER),
-                .addHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW)
+                new BasicHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW))) {
-                .execute();
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
-        assertThat(response.getStatusCode(), is(200));
+        }
     }
 
     public void testTransportClient() throws Exception {

elasticsearch/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java

@@ -7,9 +7,9 @@ package org.elasticsearch.smoketest;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
@@ -19,6 +19,7 @@ import java.io.IOException;
 
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 
+
 public class GraphWithSecurityIT extends ESRestTestCase {
 
     private final static String TEST_ADMIN_USERNAME = "test_admin";

elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java

@@ -14,7 +14,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
-import org.elasticsearch.test.rest.client.RestClient;
+import org.elasticsearch.test.rest.client.RestTestClient;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -65,9 +65,9 @@ public class SmokeTestPluginsSslIT extends ESRestTestCase {
         String token = basicAuthHeaderValue(USER, new SecuredString(PASS.toCharArray()));
         return Settings.builder()
                 .put(ThreadContext.PREFIX + ".Authorization", token)
-                .put(RestClient.PROTOCOL, "https")
+                .put(RestTestClient.PROTOCOL, "https")
-                .put(RestClient.TRUSTSTORE_PATH, keyStore)
+                .put(RestTestClient.TRUSTSTORE_PATH, keyStore)
-                .put(RestClient.TRUSTSTORE_PASSWORD, KEYSTORE_PASS)
+                .put(RestTestClient.TRUSTSTORE_PASSWORD, KEYSTORE_PASS)
                 .build();
     }
 }

elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java

@@ -5,22 +5,19 @@
  */
 package org.elasticsearch.smoketest;
 
-import java.io.IOException;
-import java.net.URI;
-import java.net.URL;
-
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+
 public abstract class WatcherRestTestCase extends ESRestTestCase {
 
     public WatcherRestTestCase(@Name("yaml") RestTestCandidate testCandidate) {
@@ -34,19 +31,11 @@ public abstract class WatcherRestTestCase extends ESRestTestCase {
 
     @Before
     public void startWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap());
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_start", null, null));
-            client.execute(request);
-        }
     }
 
     @After
     public void stopWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        getAdminExecutionContext().callApi("xpack.watcher.stop", emptyMap(), emptyList(), emptyMap());
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_stop", null, null));
-            client.execute(request);
-        }
     }
 }

elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java

@@ -7,10 +7,6 @@ package org.elasticsearch.smoketest;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
@@ -18,8 +14,9 @@ import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
-import java.net.URI;
+
-import java.net.URL;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 
 public abstract class WatcherRestTestCase extends ESRestTestCase {
 
@@ -34,19 +31,11 @@ public abstract class WatcherRestTestCase extends ESRestTestCase {
 
     @Before
     public void startWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap());
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_start", null, null));
-            client.execute(request);
-        }
     }
 
     @After
     public void stopWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        getAdminExecutionContext().callApi("xpack.watcher.stop", emptyMap(), emptyList(), emptyMap());
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_stop", null, null));
-            client.execute(request);
-        }
     }
 }

elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/10_webhook.yaml

@@ -55,7 +55,7 @@
       catch: request_timeout
       cluster.health:
         wait_for_nodes: 99
-        timeout: 10s
+        timeout: 5s
   - match: { "timed_out": true }
 
   - do:

elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/20_array_access.yaml

@@ -53,7 +53,7 @@
       catch: request_timeout
       cluster.health:
         wait_for_nodes: 99
-        timeout: 10s
+        timeout: 5s
   - match: { "timed_out": true }
 
   - do:

elasticsearch/qa/smoke-test-watcher-with-painless/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java

@@ -5,22 +5,19 @@
  */
 package org.elasticsearch.smoketest;
 
-import java.io.IOException;
-import java.net.URI;
-import java.net.URL;
-
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+
 public abstract class WatcherRestTestCase extends ESRestTestCase {
 
     public WatcherRestTestCase(@Name("yaml") RestTestCandidate testCandidate) {
@@ -34,19 +31,11 @@ public abstract class WatcherRestTestCase extends ESRestTestCase {
 
     @Before
     public void startWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap());
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_start", null, null));
-            client.execute(request);
-        }
     }
 
     @After
     public void stopWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        getAdminExecutionContext().callApi("xpack.watcher.stop", emptyMap(), emptyList(), emptyMap());
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_stop", null, null));
-            client.execute(request);
-        }
     }
 }

elasticsearch/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/WatcherWithSecurityIT.java

@@ -5,28 +5,26 @@
  */
 package org.elasticsearch.smoketest;
 
-import java.io.IOException;
-import java.net.URI;
-import java.net.URL;
-
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
-import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
 import org.junit.After;
 import org.junit.Before;
 
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 
+
 public class WatcherWithSecurityIT extends ESRestTestCase {
 
     private final static String TEST_ADMIN_USERNAME = "test_admin";
@@ -43,24 +41,12 @@ public class WatcherWithSecurityIT extends ESRestTestCase {
 
     @Before
     public void startWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap());
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_start", null, null));
-            String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecuredString(TEST_ADMIN_PASSWORD.toCharArray()));
-            request.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, token);
-            client.execute(request);
-        }
     }
 
     @After
     public void stopWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        getAdminExecutionContext().callApi("xpack.watcher.stop", emptyMap(), emptyList(), emptyMap());
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_stop", null, null));
-            String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecuredString(TEST_ADMIN_PASSWORD.toCharArray()));
-            request.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, token);
-            client.execute(request);
-        }
     }
 
     @Override

elasticsearch/x-pack/build.gradle

@@ -29,7 +29,7 @@ dependencies {
   compile 'com.unboundid:unboundid-ldapsdk:2.3.8'
   compile 'org.bouncycastle:bcprov-jdk15on:1.54'
   compile 'org.bouncycastle:bcpkix-jdk15on:1.54'
-  testCompile 'com.google.jimfs:jimfs:1.0'
+  testCompile 'com.google.jimfs:jimfs:1.1'
 
   // watcher deps
   compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239'

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java

@@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.http.HttpInfo;
 import org.elasticsearch.index.Index;
@@ -112,7 +113,7 @@ public class ClusterStatsResolverTests extends MonitoringIndexNameResolverTestCa
                 Settings.EMPTY, DummyOsInfo.INSTANCE, new ProcessInfo(randomInt(), randomBoolean()), JvmInfo.jvmInfo(),
                 new ThreadPoolInfo(Collections.singletonList(new ThreadPool.Info("test_threadpool", ThreadPool.ThreadPoolType.FIXED, 5))),
                 new TransportInfo(transportAddress, Collections.emptyMap()), new HttpInfo(transportAddress, randomLong()),
-                new PluginsAndModules(), new IngestInfo(Collections.emptyList()));
+                new PluginsAndModules(), new IngestInfo(Collections.emptyList()), new ByteSizeValue(randomIntBetween(1, 1024)));
 
     }
 

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java

@@ -43,6 +43,7 @@ public class NodeStatsTests extends MarvelIntegTestCase {
         wipeMarvelIndices();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/2588")
     public void testNodeStats() throws Exception {
         logger.debug("--> creating some indices for future node stats");
         final int numDocs = between(50, 150);

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/security/MarvelSettingsFilterTests.java

@@ -5,21 +5,18 @@
  */
 package org.elasticsearch.marvel.security;
 
-import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.Header;
-import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.marvel.MonitoringSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.hamcrest.Matchers;
-import org.junit.After;
 
-import java.io.IOException;
+import java.util.Collections;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
@@ -29,13 +26,6 @@ import static org.hamcrest.CoreMatchers.nullValue;
 
 public class MarvelSettingsFilterTests extends MarvelIntegTestCase {
 
-    private CloseableHttpClient httpClient = HttpClients.createDefault();
-
-    @After
-    public void cleanup() throws IOException {
-        httpClient.close();
-    }
-
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
@@ -53,46 +43,35 @@ public class MarvelSettingsFilterTests extends MarvelIntegTestCase {
     }
 
     public void testGetSettingsFiltered() throws Exception {
-        String body = executeRequest("GET", "/_nodes/settings", null, null).getBody();
+        Header[] headers;
-        Map<String, Object> response = JsonXContent.jsonXContent.createParser(body).map();
+        if (securityEnabled) {
-        Map<String, Object> nodes = (Map<String, Object>) response.get("nodes");
+            headers = new Header[] {
-        for (Object node : nodes.values()) {
+                    new BasicHeader(BASIC_AUTH_HEADER,
-            Map<String, Object> settings = (Map<String, Object>) ((Map<String, Object>) node).get("settings");
+                            basicAuthHeaderValue(SecuritySettings.TEST_USERNAME,
-
+                                    new SecuredString(SecuritySettings.TEST_PASSWORD.toCharArray())))};
-            assertThat(extractValue("xpack.monitoring.agent.exporters._http.type", settings), Matchers.<Object>equalTo("http"));
+        } else {
-            assertThat(extractValue("xpack.monitoring.agent.exporters._http.enabled", settings), Matchers.<Object>equalTo("false"));
+            headers = new Header[0];
-            assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.username");
+        }
-            assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.password");
+        try (Response response = getRestClient().performRequest("GET", "/_nodes/settings",
-            assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.path");
+                Collections.emptyMap(), null, headers)) {
-            assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.password");
+            Map<String, Object> responseMap = JsonXContent.jsonXContent.createParser(response.getEntity().getContent()).map();
-            assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.hostname_verification");
+            @SuppressWarnings("unchecked")
+            Map<String, Object> nodes = (Map<String, Object>) responseMap.get("nodes");
+            for (Object node : nodes.values()) {
+                @SuppressWarnings("unchecked")
+                Map<String, Object> settings = (Map<String, Object>) ((Map<String, Object>) node).get("settings");
+                assertThat(extractValue("xpack.monitoring.agent.exporters._http.type", settings), Matchers.<Object>equalTo("http"));
+                assertThat(extractValue("xpack.monitoring.agent.exporters._http.enabled", settings), Matchers.<Object>equalTo("false"));
+                assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.username");
+                assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.password");
+                assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.path");
+                assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.password");
+                assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.hostname_verification");
+            }
         }
     }
 
     private void assertNullSetting(Map<String, Object> settings, String setting) {
         assertThat(extractValue(setting, settings), nullValue());
     }
-
-    protected HttpResponse executeRequest(String method, String path, String body, Map<String, String> params) throws IOException {
-        HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class,
-                internalCluster().getMasterName());
-        HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient)
-                .httpTransport(httpServerTransport)
-                .method(method)
-                .path(path);
-
-        if (params != null) {
-            for (Map.Entry<String, String> entry : params.entrySet()) {
-                requestBuilder.addParam(entry.getKey(), entry.getValue());
-            }
-        }
-        if (body != null) {
-            requestBuilder.body(body);
-        }
-        if (securityEnabled) {
-            requestBuilder.addHeader(BASIC_AUTH_HEADER,
-                    basicAuthHeaderValue(SecuritySettings.TEST_USERNAME, new SecuredString(SecuritySettings.TEST_PASSWORD.toCharArray())));
-        }
-        return requestBuilder.execute();
-    }
 }

elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/RoleDescriptor.java

@@ -235,10 +235,6 @@ public class RoleDescriptor implements ToXContent {
                 }
             } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.PRIVILEGES)) {
                 privileges = readStringArray(roleName, parser, true);
-                if (names.length == 0) {
-                    throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. [{}] cannot be an empty " +
-                            "array", roleName, currentFieldName);
-                }
             } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.FIELDS)) {
                 fields = readStringArray(roleName, parser, true);
             } else {

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java

@@ -5,16 +5,18 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.HttpEntity;
-import org.apache.http.impl.client.HttpClients;
+import org.apache.http.StatusLine;
-import org.elasticsearch.http.HttpServerTransport;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.xpack.security.authc.support.Hasher;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
-import org.junit.After;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -32,19 +34,16 @@ public abstract class AbstractPrivilegeTestCase extends SecurityIntegTestCase {
 
     protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray())));
 
-    private CloseableHttpClient httpClient = HttpClients.createDefault();
-
-    @After
-    public void cleanup() throws IOException {
-        httpClient.close();
-    }
-
     protected void assertAccessIsAllowed(String user, String method, String uri, String body,
                                          Map<String, String> params) throws IOException {
-        HttpResponse response = executeRequest(user, method, uri, body, params);
+        try (Response response = getRestClient().performRequest(method, uri, params, entityOrNull(body),
-        String message = String.format(Locale.ROOT, "%s %s: Expected no error got %s %s with body %s", method, uri,
+                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                response.getStatusCode(), response.getReasonPhrase(), response.getBody());
+                        UsernamePasswordToken.basicAuthHeaderValue(user, new SecuredString("passwd".toCharArray()))))) {
-        assertThat(message, response.getStatusCode(), is(not(greaterThanOrEqualTo(400))));
+            StatusLine statusLine = response.getStatusLine();
+            String message = String.format(Locale.ROOT, "%s %s: Expected no error got %s %s with body %s", method, uri,
+                    statusLine.getStatusCode(), statusLine.getReasonPhrase(), EntityUtils.toString(response.getEntity()));
+            assertThat(message, statusLine.getStatusCode(), is(not(greaterThanOrEqualTo(400))));
+        }
     }
 
     protected void assertAccessIsAllowed(String user, String method, String uri, String body) throws IOException {
@@ -65,28 +64,24 @@ public abstract class AbstractPrivilegeTestCase extends SecurityIntegTestCase {
 
     protected void assertAccessIsDenied(String user, String method, String uri, String body,
                                         Map<String, String> params) throws IOException {
-        HttpResponse response = executeRequest(user, method, uri, body, params);
+        try {
-        String message = String.format(Locale.ROOT, "%s %s body %s: Expected 403, got %s %s with body %s", method, uri, body,
+            getRestClient().performRequest(method, uri, params, entityOrNull(body),
-                response.getStatusCode(), response.getReasonPhrase(), response.getBody());
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-        assertThat(message, response.getStatusCode(), is(403));
+                            UsernamePasswordToken.basicAuthHeaderValue(user, new SecuredString("passwd".toCharArray()))));
+            fail("request should have failed");
+        } catch(ResponseException e) {
+            StatusLine statusLine = e.getResponse().getStatusLine();
+            String message = String.format(Locale.ROOT, "%s %s body %s: Expected 403, got %s %s with body %s", method, uri, body,
+                    statusLine.getStatusCode(), statusLine.getReasonPhrase(), e.getResponseBody());
+            assertThat(message, statusLine.getStatusCode(), is(403));
+        }
     }
 
-    protected HttpResponse executeRequest(String user, String method, String uri, String body,
+    private static HttpEntity entityOrNull(String body) {
-                                          Map<String, String> params) throws IOException {
+        HttpEntity entity = null;
-        HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class);
-
-        HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport);
-        requestBuilder.path(uri);
-        requestBuilder.method(method);
-        for (Map.Entry<String, String> entry : params.entrySet()) {
-            requestBuilder.addParam(entry.getKey(), entry.getValue());
-        }
         if (body != null) {
-            requestBuilder.body(body);
+            entity = new StringEntity(body, RestClient.JSON_CONTENT_TYPE);
         }
-        requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(user,
+        return entity;
-                new SecuredString("passwd".toCharArray())));
-        return requestBuilder.execute();
     }
-
 }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java

@@ -5,9 +5,15 @@
  */
 package org.elasticsearch.integration;
 
+import org.apache.http.Header;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.security.Security;
@@ -15,10 +21,10 @@ import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.xpack.XPackPlugin;
 
 import java.io.IOException;
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -49,8 +55,8 @@ public class BulkUpdateTests extends SecurityIntegTestCase {
                 .get().isCreated();
         assertThat(created, is(false));
         getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").setFields("test", "not test").get();
-        assertThat((String) getResponse.getField("test").getValue(), equalTo("test"));
+        assertThat(getResponse.getField("test").getValue(), equalTo("test"));
-        assertThat((String) getResponse.getField("not test").getValue(), equalTo("not test"));
+        assertThat(getResponse.getField("not test").getValue(), equalTo("not test"));
 
         // this part is important. Without this, the document may be read from the translog which would bypass the bug where
         // FLS kicks in because the request can't be found and only returns meta fields
@@ -62,43 +68,61 @@ public class BulkUpdateTests extends SecurityIntegTestCase {
         assertThat(((UpdateResponse)response.getItems()[0].getResponse()).isCreated(), is(false));
         getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").
                 setFields("test", "not test", "bulk updated").get();
-        assertThat((String) getResponse.getField("test").getValue(), equalTo("test"));
+        assertThat(getResponse.getField("test").getValue(), equalTo("test"));
-        assertThat((String) getResponse.getField("not test").getValue(), equalTo("not test"));
+        assertThat(getResponse.getField("not test").getValue(), equalTo("not test"));
-        assertThat((String) getResponse.getField("bulk updated").getValue(), equalTo("bulk updated"));
+        assertThat(getResponse.getField("bulk updated").getValue(), equalTo("bulk updated"));
     }
 
     public void testThatBulkUpdateDoesNotLoseFieldsHttp() throws IOException {
         final String path = "/index1/type/1";
-        final String basicAuthHeader = UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
+        final Header basicAuthHeader = new BasicHeader("Authorization",
-                new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()));
+                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
+                        new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())));
 
-        httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("PUT").body("{\"test\":\"test\"}").execute();
+        StringEntity body = new StringEntity("{\"test\":\"test\"}", RestClient.JSON_CONTENT_TYPE);
-        HttpResponse response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute();
+        try (Response response = getRestClient().performRequest("PUT", path, Collections.emptyMap(), body, basicAuthHeader)) {
-        assertThat(response.getBody(), containsString("\"test\":\"test\""));
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(201));
+        }
+
+        try (Response response = getRestClient().performRequest("GET", path, Collections.emptyMap(), null, basicAuthHeader)) {
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+            assertThat(EntityUtils.toString(response.getEntity()), containsString("\"test\":\"test\""));
+        }
 
         if (randomBoolean()) {
             flushAndRefresh();
         }
 
         //update with new field
-        httpClient().path(path + "/_update").addHeader("Authorization", basicAuthHeader).method("POST").
+        body = new StringEntity("{\"doc\": {\"not test\": \"not test\"}}", RestClient.JSON_CONTENT_TYPE);
-                body("{\"doc\": {\"not test\": \"not test\"}}").execute();
+        try (Response response = getRestClient().performRequest("POST", path + "/_update",
-        response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute();
+                Collections.emptyMap(), body, basicAuthHeader)) {
-        assertThat(response.getBody(), containsString("\"test\":\"test\""));
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-        assertThat(response.getBody(), containsString("\"not test\":\"not test\""));
+        }
+
+        try (Response response = getRestClient().performRequest("GET", path, Collections.emptyMap(), null, basicAuthHeader)) {
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+            String responseBody = EntityUtils.toString(response.getEntity());
+            assertThat(responseBody, containsString("\"test\":\"test\""));
+            assertThat(responseBody, containsString("\"not test\":\"not test\""));
+        }
 
         // this part is important. Without this, the document may be read from the translog which would bypass the bug where
         // FLS kicks in because the request can't be found and only returns meta fields
         flushAndRefresh();
 
-        // update with bulk
+        body = new StringEntity("{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n" +
-        httpClient().path("/_bulk").addHeader("Authorization", basicAuthHeader).method("POST")
+                "{\"doc\": {\"bulk updated\":\"bulk updated\"}}\n", RestClient.JSON_CONTENT_TYPE);
-                .body("{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n{\"doc\": {\"bulk updated\":\"bulk " +
+        try (Response response = getRestClient().performRequest("POST", "/_bulk",
-                        "updated\"}}\n")
+                Collections.emptyMap(), body, basicAuthHeader)) {
-                .execute();
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-        response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute();
+        }
-        assertThat(response.getBody(), containsString("\"test\":\"test\""));
+
-        assertThat(response.getBody(), containsString("\"not test\":\"not test\""));
+        try (Response response = getRestClient().performRequest("GET", path, Collections.emptyMap(), null, basicAuthHeader)) {
-        assertThat(response.getBody(), containsString("\"bulk updated\":\"bulk updated\""));
+            String responseBody = EntityUtils.toString(response.getEntity());
+            assertThat(responseBody, containsString("\"test\":\"test\""));
+            assertThat(responseBody, containsString("\"not test\":\"not test\""));
+            assertThat(responseBody, containsString("\"bulk updated\":\"bulk updated\""));
+        }
     }
 }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java

@@ -5,14 +5,15 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
-import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.xpack.security.user.User;
+import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheResponse;
 import org.elasticsearch.xpack.security.authc.Realm;
@@ -22,10 +23,7 @@ import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.authc.support.SecuredStringTests;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.security.client.SecurityClient;
-import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.xpack.security.user.User;
-import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.junit.BeforeClass;
 
 import java.util.ArrayList;
@@ -39,14 +37,10 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
-/**
- *
- */
 public class ClearRealmsCacheTests extends SecurityIntegTestCase {
     private static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray())));
 
@@ -168,21 +162,12 @@ public class ClearRealmsCacheTests extends SecurityIntegTestCase {
         }
 
         static void executeHttpRequest(String path, Map<String, String> params) throws Exception {
-            try (CloseableHttpClient client = HttpClients.createDefault()) {
+            try (Response response = getRestClient().performRequest("POST", path, params, null,
-                HttpRequestBuilder requestBuilder = new HttpRequestBuilder(client)
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        .httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class))
+                            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                        .method("POST")
+                                    new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) {
-                        .path(path);
+                assertNotNull(response.getEntity());
-                for (Map.Entry<String, String> entry : params.entrySet()) {
+                assertTrue(EntityUtils.toString(response.getEntity()).contains("cluster_name"));
-                    requestBuilder.addParam(entry.getKey(), entry.getValue());
-                }
-                requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                                new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())));
-                HttpResponse response = requestBuilder.execute();
-                assertThat(response.hasBody(), is(true));
-                String body = response.getBody();
-                assertThat(body.contains("cluster_name"), is(true));
             }
         }
     }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java

@@ -5,9 +5,11 @@
  */
 package org.elasticsearch.integration;
 
+import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
@@ -23,11 +25,11 @@ import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 import org.elasticsearch.xpack.security.client.SecurityClient;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -81,7 +83,7 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
         logger.debug("using poller interval [{}]", pollerInterval);
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(NativeRolesStore.POLL_INTERVAL_SETTING.getKey(), pollerInterval)
+                .put(NativeRolesStore.POLL_INTERVAL_SETTING.getKey(), pollerInterval.getStringRep())
                 .put(NetworkModule.HTTP_ENABLED.getKey(), true)
                 .build();
     }
@@ -128,7 +130,7 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
         final boolean useHttp = randomBoolean();
         final boolean clearAll = randomBoolean();
         logger.debug("--> starting to clear roles. using http [{}] clearing all [{}]", useHttp, clearAll);
-        String[] rolesToClear = clearAll ? (randomBoolean() ? roles : null) : toModify.toArray(Strings.EMPTY_ARRAY);
+        String[] rolesToClear = clearAll ? (randomBoolean() ? roles : null) : toModify.toArray(new String[toModify.size()]);
         if (useHttp) {
             String path;
             if (rolesToClear == null) {
@@ -136,12 +138,12 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
             } else {
                 path = "/_xpack/security/role/" + Strings.arrayToCommaDelimitedString(rolesToClear) + "/_clear_cache";
             }
-            HttpResponse response = httpClient().path(path).method("POST")
+            try (Response response = getRestClient().performRequest("POST", path, Collections.emptyMap(), null,
-                    .addHeader("Authorization",
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
                             UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                                    new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())))
+                                    new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) {
-                    .execute();
+                assertThat(response.getStatusLine().getStatusCode(), is(RestStatus.OK.getStatus()));
-            assertThat(response.getStatusCode(), is(RestStatus.OK.getStatus()));
+            }
         } else {
             securityClient.prepareClearRolesCache().names(rolesToClear).get();
         }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java

@@ -5,20 +5,21 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.lucene.util.LuceneTestCase.BadApple;
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
+import org.elasticsearch.xpack.security.authc.support.SecuredString;
+import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.junit.Before;
 
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 
 import static java.util.Collections.singletonMap;
-import static org.hamcrest.Matchers.containsString;
+import static org.apache.lucene.util.LuceneTestCase.BadApple;
 import static org.hamcrest.Matchers.is;
 
 //test is just too slow, please fix it to not be sleep-based
@@ -303,8 +304,14 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
     }
 
     public void testThatUnknownUserIsRejectedProperly() throws Exception {
-        HttpResponse response = executeRequest("idonotexist", "GET", "/", null, new HashMap<>());
+        try {
-        assertThat(response.getStatusCode(), is(401));
+            getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                            UsernamePasswordToken.basicAuthHeaderValue("idonotexist", new SecuredString("passwd".toCharArray()))));
+            fail("request should have failed");
+        } catch(ResponseException e) {
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
+        }
     }
 
     private void assertUserExecutes(String user, String action, String index, boolean userIsAllowed) throws Exception {

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/LicensingTests.java

@@ -14,6 +14,8 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -194,13 +196,20 @@ public class LicensingTests extends SecurityIntegTestCase {
     }
 
     public void testRestAuthenticationByLicenseType() throws Exception {
-        // the default of the licensing tests is basic
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
-        assertThat(httpClient().path("/").execute().getStatusCode(), is(200));
+            // the default of the licensing tests is basic
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+        }
 
         // generate a new license with a mode that enables auth
         OperationMode mode = randomFrom(OperationMode.GOLD, OperationMode.TRIAL, OperationMode.PLATINUM, OperationMode.STANDARD);
         enableLicensing(mode);
-        assertThat(httpClient().path("/").execute().getStatusCode(), is(401));
+        try {
+            getRestClient().performRequest("GET", "/", Collections.emptyMap(), null);
+            fail("request should have failed");
+        } catch(ResponseException e) {
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
+        }
     }
 
     public void testTransportClientAuthenticationByLicenseType() throws Exception {

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginEnabledDisabledTests.java

@@ -19,7 +19,6 @@ import org.junit.After;
 import org.junit.BeforeClass;
 
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 
 /**

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java

@@ -5,23 +5,21 @@
  */
 package org.elasticsearch.xpack.security;
 
-import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
-import org.apache.http.impl.client.HttpClients;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 
 import java.io.IOException;
+import java.util.Collections;
 
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED;
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.is;
 
 public class SecurityPluginTests extends SecurityIntegTestCase {
@@ -35,24 +33,20 @@ public class SecurityPluginTests extends SecurityIntegTestCase {
     }
 
     public void testThatPluginIsLoaded() throws IOException {
-        HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class);
+        try {
-        try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
             logger.info("executing unauthorized request to /_xpack info");
-            HttpResponse response = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport)
+            getRestClient().performRequest("GET", "/_xpack", Collections.emptyMap(), null);
-                    .method("GET")
+            fail("request should have failed");
-                    .path("/_xpack")
+        } catch(ResponseException e) {
-                    .execute();
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(UNAUTHORIZED.getStatus()));
-            assertThat(response.getStatusCode(), is(UNAUTHORIZED.getStatus()));
+        }
 
-            logger.info("executing authorized request to /_xpack infos");
+        logger.info("executing authorized request to /_xpack infos");
-            response = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport)
+        try (Response response = getRestClient().performRequest("GET", "/_xpack", Collections.emptyMap(), null,
-                    .method("GET")
+                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    .path("/_xpack")
-                    .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
                         basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                            new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())))
+                                new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) {
-                    .execute();
+            assertThat(response.getStatusLine().getStatusCode(), is(OK.getStatus()));
-            assertThat(response.getStatusCode(), is(OK.getStatus()));
         }
     }
 }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java

@@ -5,10 +5,13 @@
  */
 package org.elasticsearch.xpack.security.authc;
 
+import org.apache.http.message.BasicHeader;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
@@ -20,7 +23,6 @@ import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.security.transport.netty.SecurityNettyTransport;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Collections;
@@ -115,30 +117,36 @@ public class RunAsIntegTests extends SecurityIntegTestCase {
 
     public void testUserImpersonationUsingHttp() throws Exception {
         // use the transport client user and try to run as
-        HttpResponse response = httpClient().method("GET")
+        try {
-                .path("/_nodes")
+            getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null,
-                .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER,
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD)))
+                            UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER,
-                .addHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, SecuritySettingsSource.DEFAULT_USER_NAME)
+                                    SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))),
-                .execute();
+                    new BasicHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, SecuritySettingsSource.DEFAULT_USER_NAME));
-        assertThat(response.getStatusCode(), is(403));
+            fail("request should have failed");
+        } catch(ResponseException e) {
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
+        }
 
-        //the run as user shouldn't have access to the nodes api
+        try {
-        response = httpClient().method("GET")
+            //the run as user shouldn't have access to the nodes api
-                .path("/_nodes")
+            getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null,
-                .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD)))
+                            UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                .execute();
+                                    SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))));
-        assertThat(response.getStatusCode(), is(403));
+            fail("request should have failed");
+        } catch(ResponseException e) {
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
+        }
 
         // but when running as a different user it should work
-        response = httpClient().method("GET")
+        try (Response response = getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null,
-                .path("/_nodes")
+                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
+                        UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                        SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD)))
+                                SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))),
-                .addHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, SecuritySettingsSource.DEFAULT_USER_NAME)
+                new BasicHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, SecuritySettingsSource.DEFAULT_USER_NAME))) {
-                .execute();
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
-        assertThat(response.getStatusCode(), is(200));
+        }
     }
 
     public void testEmptyUserImpersonationHeader() throws Exception {
@@ -164,13 +172,16 @@ public class RunAsIntegTests extends SecurityIntegTestCase {
     }
 
     public void testEmptyHeaderUsingHttp() throws Exception {
-        HttpResponse response = httpClient().method("GET")
+        try {
-                .path("/_nodes")
+            getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null,
-                .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD)))
+                            UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                .addHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, "")
+                            SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))),
-                .execute();
+                    new BasicHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, ""));
-        assertThat(response.getStatusCode(), is(401));
+            fail("request should have failed");
+        } catch(ResponseException e) {
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
+        }
     }
 
     public void testNonExistentRunAsUser() throws Exception {
@@ -196,13 +207,16 @@ public class RunAsIntegTests extends SecurityIntegTestCase {
     }
 
     public void testNonExistentRunAsUserUsingHttp() throws Exception {
-        HttpResponse response = httpClient().method("GET")
+        try {
-                .path("/_nodes")
+            getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null,
-                .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                        SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD)))
+                            UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER,
-                .addHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, "idontexist")
+                            SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))),
-                .execute();
+                    new BasicHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, "idontexist"));
-        assertThat(response.getStatusCode(), is(403));
+            fail("request should have failed");
+        } catch (ResponseException e) {
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
+        }
     }
 
     // build our own here to better mimic an actual client...

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java

@@ -7,11 +7,14 @@ package org.elasticsearch.xpack.security.authc.pki;
 
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
@@ -20,8 +23,6 @@ import org.elasticsearch.xpack.security.transport.netty.SecurityNettyHttpServerT
 import org.elasticsearch.xpack.security.transport.netty.SecurityNettyTransport;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.junit.BeforeClass;
@@ -34,6 +35,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyStore;
 import java.security.SecureRandom;
+import java.util.Collections;
 
 import static org.elasticsearch.test.SecuritySettingsSource.DEFAULT_PASSWORD;
 import static org.elasticsearch.test.SecuritySettingsSource.DEFAULT_USER_NAME;
@@ -78,23 +80,21 @@ public class PkiOptionalClientAuthTests extends SecurityIntegTestCase {
     }
 
     public void testRestClientWithoutClientCertificate() throws Exception {
-        HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class);
+        CloseableHttpClient httpClient = HttpClients.custom().setSSLContext(getSSLContext()).build();
+        try (RestClient restClient = createRestClient(httpClient, "https")) {
+            try {
+                restClient.performRequest("GET", "_nodes", Collections.emptyMap(), null);
+                fail("request should have failed");
+            } catch(ResponseException e) {
+                assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
+            }
 
-        try (CloseableHttpClient httpClient = HttpClients.custom().setSslcontext(getSSLContext()).build()) {
+            try (Response response = restClient.performRequest("GET", "_nodes", Collections.emptyMap(), null,
-            HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient)
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    .host("localhost")
+                            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                    .port(((InetSocketTransportAddress)httpServerTransport.boundAddress().publishAddress()).address().getPort())
+                                    new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) {
-                    .protocol("https")
+                assertThat(response.getStatusLine().getStatusCode(), is(200));
-                    .method("GET")
+            }
-                    .path("/_nodes");
-            HttpResponse response = requestBuilder.execute();
-            assertThat(response.getStatusCode(), is(401));
-
-            requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                            new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())));
-            response = requestBuilder.execute();
-            assertThat(response.getStatusCode(), is(200));
         }
     }
 

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutClientAuthenticationTests.java

@@ -8,27 +8,27 @@ package org.elasticsearch.xpack.security.authc.pki;
 
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.security.transport.SSLClientAuth;
 import org.elasticsearch.xpack.security.transport.netty.SecurityNettyHttpServerTransport;
 import org.elasticsearch.xpack.security.transport.netty.SecurityNettyTransport;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
 import java.security.SecureRandom;
 import java.security.cert.X509Certificate;
+import java.util.Collections;
 import java.util.Locale;
 
 import static org.hamcrest.Matchers.is;
@@ -77,21 +77,16 @@ public class PkiWithoutClientAuthenticationTests extends SecurityIntegTestCase {
     }
 
     public void testThatHttpWorks() throws Exception {
-        HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class);
         SSLContext sc = SSLContext.getInstance("SSL");
         sc.init(null, trustAllCerts, new SecureRandom());
-        try (CloseableHttpClient httpClient = HttpClients.custom().setSslcontext(sc).build()) {
+        CloseableHttpClient httpClient = HttpClients.custom().setSSLContext(sc).build();
-            HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient)
+        try (RestClient restClient =  createRestClient(httpClient, "https")) {
-                    .host("localhost")
+            try (Response response = restClient.performRequest("GET", "/_nodes", Collections.emptyMap(), null,
-                    .port(((InetSocketTransportAddress)httpServerTransport.boundAddress().publishAddress()).address().getPort())
+                    new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    .protocol("https")
+                            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                    .method("GET")
+                                    new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) {
-                    .path("/_nodes");
+                assertThat(response.getStatusLine().getStatusCode(), is(200));
-            requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+            }
-                    UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                            new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())));
-            HttpResponse response = requestBuilder.execute();
-            assertThat(response.getStatusCode(), is(200));
         }
     }
 }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutSSLTests.java

@@ -5,19 +5,18 @@
  */
 package org.elasticsearch.xpack.security.authc.pki;
 
-import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
-import org.apache.http.impl.client.HttpClients;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.http.HttpServerTransport;
-import org.elasticsearch.xpack.security.authc.support.SecuredString;
-import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
+import org.elasticsearch.xpack.security.authc.support.SecuredString;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
+import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
+
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.is;
 
@@ -44,17 +43,11 @@ public class PkiWithoutSSLTests extends SecurityIntegTestCase {
     }
 
     public void testThatHttpWorks() throws Exception {
-        HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class);
+        try (Response response = getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null,
-        try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+                new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-            HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient)
+                        UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                    .httpTransport(httpServerTransport)
+                                new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) {
-                    .method("GET")
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
-                    .path("/_nodes");
-            requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
-                    UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                            new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())));
-            HttpResponse response = requestBuilder.execute();
-            assertThat(response.getStatusCode(), is(200));
         }
     }
 }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java

@@ -64,6 +64,7 @@ import java.util.Collections;
 import java.util.IdentityHashMap;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonMap;
@@ -94,7 +95,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
                 Collections.emptyMap(), Collections.emptyMap());
         SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
         mapperService = new MapperService(indexSettings, analysisService, similarityService,
-                new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry(), () -> null);
+                new IndicesModule(new NamedWriteableRegistry(), emptyList()).getMapperRegistry(), () -> null);
 
         ShardId shardId = new ShardId(index, 0);
         licenseState = mock(SecurityLicenseState.class);

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java

@@ -5,6 +5,10 @@
  */
 package org.elasticsearch.xpack.security.rest.action;
 
+import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
@@ -12,10 +16,10 @@ import org.elasticsearch.xpack.security.authz.InternalAuthorizationService;
 import org.elasticsearch.xpack.security.user.AnonymousUser;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.test.rest.json.JsonPath;
 import org.junit.BeforeClass;
 
+import java.util.Collections;
 import java.util.List;
 
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
@@ -47,32 +51,40 @@ public class RestAuthenticateActionTests extends SecurityIntegTestCase {
     }
 
     public void testAuthenticateApi() throws Exception {
-        HttpResponse response = httpClient().method("GET").path("/_xpack/security/_authenticate")
+        try (Response response = getRestClient().performRequest(
-                .addHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
+                "GET", "/_xpack/security/_authenticate", Collections.emptyMap(), null,
-                        new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())))
+                new BasicHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME,
-                .execute();
+                        new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) {
-
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
-        assertThat(response.getStatusCode(), is(200));
+            JsonPath jsonPath = new JsonPath(EntityUtils.toString(response.getEntity()));
-        JsonPath jsonPath = new JsonPath(response.getBody());
+            assertThat(jsonPath.evaluate("username").toString(), equalTo(SecuritySettingsSource.DEFAULT_USER_NAME));
-        assertThat(jsonPath.evaluate("username").toString(), equalTo(SecuritySettingsSource.DEFAULT_USER_NAME));
+            @SuppressWarnings("unchecked")
-        List<String> roles = (List<String>) jsonPath.evaluate("roles");
+            List<String> roles = (List<String>) jsonPath.evaluate("roles");
-        assertThat(roles.size(), is(1));
+            assertThat(roles.size(), is(1));
-        assertThat(roles, contains(SecuritySettingsSource.DEFAULT_ROLE));
+            assertThat(roles, contains(SecuritySettingsSource.DEFAULT_ROLE));
+        }
     }
 
     public void testAuthenticateApiWithoutAuthentication() throws Exception {
-        HttpResponse response = httpClient().method("GET").path("/_xpack/security/_authenticate")
+        try (Response response = getRestClient().performRequest("GET", "/_xpack/security/_authenticate",
-                .execute();
+                Collections.emptyMap(), null)) {
-
+            if (anonymousEnabled) {
-        if (anonymousEnabled) {
+                assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getStatusCode(), is(200));
+                JsonPath jsonPath = new JsonPath(EntityUtils.toString(response.getEntity()));
-            JsonPath jsonPath = new JsonPath(response.getBody());
+                assertThat(jsonPath.evaluate("username").toString(), equalTo("anon"));
-            assertThat(jsonPath.evaluate("username").toString(), equalTo("anon"));
+                @SuppressWarnings("unchecked")
-            List<String> roles = (List<String>) jsonPath.evaluate("roles");
+                List<String> roles = (List<String>) jsonPath.evaluate("roles");
-            assertThat(roles.size(), is(2));
+                assertThat(roles.size(), is(2));
-            assertThat(roles, contains(SecuritySettingsSource.DEFAULT_ROLE, "foo"));
+                assertThat(roles, contains(SecuritySettingsSource.DEFAULT_ROLE, "foo"));
-        } else {
+            } else {
-            assertThat(response.getStatusCode(), is(401));
+                fail("request should have failed");
+            }
+        } catch(ResponseException e) {
+            if (anonymousEnabled) {
+                fail("request should have succeeded");
+            } else {
+                assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401));
+            }
         }
     }
 }

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslClientAuthTests.java

@@ -9,20 +9,21 @@ import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLContexts;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.ssl.ClientSSLService;
 import org.elasticsearch.xpack.security.ssl.SSLConfiguration.Global;
 import org.elasticsearch.xpack.security.transport.netty.SecurityNettyHttpServerTransport;
 import org.elasticsearch.xpack.security.transport.netty.SecurityNettyTransport;
 import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.XPackPlugin;
 
@@ -30,10 +31,12 @@ import javax.net.ssl.SSLHandshakeException;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collections;
 
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.test.SecuritySettingsSource.getSSLSettingsForStore;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class SslClientAuthTests extends SecurityIntegTestCase {
     @Override
@@ -59,14 +62,8 @@ public class SslClientAuthTests extends SecurityIntegTestCase {
                 SSLContexts.createDefault(),
                 SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
 
-        CloseableHttpClient client = HttpClients.custom().setSSLSocketFactory(socketFactory).build();
+        try (RestClient restClient = createRestClient(HttpClients.custom().setSSLSocketFactory(socketFactory).build(), "https")) {
-
+            restClient.performRequest("GET", "/", Collections.emptyMap(), null);
-        try {
-            new HttpRequestBuilder(client)
-                    .httpTransport(internalCluster().getInstance(HttpServerTransport.class))
-                    .method("GET").path("/")
-                    .protocol("https")
-                    .execute();
             fail("Expected SSLHandshakeException");
         } catch (SSLHandshakeException e) {
             assertThat(e.getMessage(), containsString("unable to find valid certification path to requested target"));
@@ -85,13 +82,13 @@ public class SslClientAuthTests extends SecurityIntegTestCase {
 
         CloseableHttpClient client = HttpClients.custom().setSSLSocketFactory(socketFactory).build();
 
-        HttpResponse response = new HttpRequestBuilder(client)
+        try (RestClient restClient = createRestClient(client, "https")) {
-                .httpTransport(internalCluster().getInstance(HttpServerTransport.class))
+            try (Response response = restClient.performRequest("GET", "/", Collections.emptyMap(), null,
-                .method("GET").path("/")
+                    new BasicHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword())))) {
-                .protocol("https")
+                assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-                .addHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword()))
+                assertThat(EntityUtils.toString(response.getEntity()), containsString("You Know, for Search"));
-                .execute();
+            }
-        assertThat(response.getBody(), containsString("You Know, for Search"));
+        }
     }
 
     public void testThatTransportWorksWithoutSslClientAuth() throws Exception {

elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java

@@ -5,25 +5,16 @@
  */
 package org.elasticsearch.xpack.security.user;
 
-import org.apache.http.client.methods.CloseableHttpResponse;
+import org.elasticsearch.client.Response;
-import org.apache.http.client.methods.HttpGet;
+import org.elasticsearch.client.ResponseException;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.xpack.security.authz.InternalAuthorizationService;
 import org.elasticsearch.test.SecurityIntegTestCase;
 
-import java.io.InputStreamReader;
+import java.util.Collections;
-import java.nio.charset.StandardCharsets;
-import java.util.Locale;
 
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -51,28 +42,22 @@ public class AnonymousUserIntegTests extends SecurityIntegTestCase {
     }
 
     public void testAnonymousViaHttp() throws Exception {
-        try (CloseableHttpClient client = HttpClients.createDefault();
+        try {
-             CloseableHttpResponse response = client.execute(new HttpGet(getNodeUrl() + "_nodes"))) {
+            getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null);
-            int statusCode = response.getStatusLine().getStatusCode();
+            fail("request should have failed");
-            String data = Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8));
+        } catch(ResponseException e) {
+            int statusCode = e.getResponse().getStatusLine().getStatusCode();
+            Response response = e.getResponse();
             if (authorizationExceptionsEnabled) {
                 assertThat(statusCode, is(403));
-                assertThat(response.getFirstHeader("WWW-Authenticate"), nullValue());
+                assertThat(response.getHeader("WWW-Authenticate"), nullValue());
-                assertThat(data, containsString("security_exception"));
+                assertThat(e.getResponseBody(), containsString("security_exception"));
             } else {
                 assertThat(statusCode, is(401));
-                assertThat(response.getFirstHeader("WWW-Authenticate"), notNullValue());
+                assertThat(response.getHeader("WWW-Authenticate"), notNullValue());
-                assertThat(response.getFirstHeader("WWW-Authenticate").getValue(), containsString("Basic"));
+                assertThat(response.getHeader("WWW-Authenticate"), containsString("Basic"));
-                assertThat(data, containsString("security_exception"));
+                assertThat(e.getResponseBody(), containsString("security_exception"));
             }
         }
     }
-
-    private String getNodeUrl() {
-        TransportAddress transportAddress =
-                randomFrom(internalCluster().getInstance(HttpServerTransport.class).boundAddress().boundAddresses());
-        assertThat(transportAddress, is(instanceOf(InetSocketTransportAddress.class)));
-        InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) transportAddress;
-        return String.format(Locale.ROOT, "http://%s:%s/", "localhost", inetSocketTransportAddress.address().getPort());
-    }
 }

elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/actions

@@ -1,4 +1,3 @@
-cluster:admin/render/template/search
 cluster:admin/repository/delete
 cluster:admin/repository/get
 cluster:admin/repository/put

elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/handlers

@@ -1,4 +1,3 @@
-cluster:admin/render/template/search
 cluster:admin/snapshot/status[nodes]
 cluster:admin/snapshot/status[nodes][n]
 cluster:admin/tasks/cancel[n]

elasticsearch/x-pack/security/src/test/resources/rest-api-spec/test/roles/10_basic.yaml

@@ -22,6 +22,21 @@
             }
   - match: { role: { created: true } }
 
+  - do:
+      xpack.security.put_role:
+        name: "backwards_role"
+        body:  >
+            {
+              "cluster": ["all"],
+              "indices": [
+                {
+                  "privileges": ["all"],
+                  "names": "*"
+                }
+              ]
+            }
+  - match: { role: { created: true } }
+
   - do:
       xpack.security.put_user:
         username: "joe"

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/http/HttpResponse.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.common.http;
 
-import com.google.common.collect.ImmutableMap;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -87,11 +86,11 @@ public class HttpResponse implements ToXContent {
      * in the payload
      */
     public Map<String, List<String>> headers() {
-        ImmutableMap.Builder<String, List<String>> builder = ImmutableMap.builder();
+        MapBuilder<String, List<String>> builder = MapBuilder.newMapBuilder();
         for (Map.Entry<String, String[]> entry : headers.entrySet()) {
             builder.put(entry.getKey().toLowerCase(Locale.ROOT), Arrays.asList(entry.getValue()));
         }
-        return builder.build();
+        return builder.immutableMap();
     }
 
     public String[] header(String header) {

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Attachment.java

@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.notification.email;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.inject.Provider;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -16,19 +17,26 @@ import javax.activation.DataHandler;
 import javax.activation.DataSource;
 import javax.activation.FileDataSource;
 import javax.mail.MessagingException;
-import javax.mail.Part;
 import javax.mail.internet.MimeBodyPart;
 import javax.mail.util.ByteArrayDataSource;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.Path;
 
+import static javax.mail.Part.ATTACHMENT;
+import static javax.mail.Part.INLINE;
+
 /**
 *
 */
 public abstract class Attachment extends BodyPartSource {
 
-    protected Attachment(String id, String name, String contentType) {
+    private final boolean inline;
+
+    protected Attachment(String id, String name, String contentType, boolean inline) {
         super(id, name, contentType);
+        this.inline = inline;
     }
 
     @Override
@@ -36,13 +44,17 @@ public abstract class Attachment extends BodyPartSource {
         MimeBodyPart part = new MimeBodyPart();
         part.setContentID(id);
         part.setFileName(name);
-        part.setDisposition(Part.ATTACHMENT);
+        part.setDisposition(inline ? INLINE : ATTACHMENT);
         writeTo(part);
         return part;
     }
 
     public abstract String type();
 
+    public boolean isInline() {
+        return inline;
+    }
+
     /**
      * intentionally not emitting path as it may come as an information leak
      */
@@ -65,22 +77,22 @@ public abstract class Attachment extends BodyPartSource {
         private final Path path;
         private final DataSource dataSource;
 
-        public File(String id, Path path) {
+        public File(String id, Path path, boolean inline) {
-            this(id, path.getFileName().toString(), path);
+            this(id, path.getFileName().toString(), path, inline);
         }
 
-        public File(String id, Path path, String contentType) {
+        public File(String id, Path path, String contentType, boolean inline) {
-            this(id, path.getFileName().toString(), path, contentType);
+            this(id, path.getFileName().toString(), path, contentType, inline);
         }
 
         @SuppressForbidden(reason = "uses toFile")
-        public File(String id, String name, Path path) {
+        public File(String id, String name, Path path, boolean inline) {
-            this(id, name, path, fileTypeMap.getContentType(path.toFile()));
+            this(id, name, path, fileTypeMap.getContentType(path.toFile()), inline);
         }
 
         @SuppressForbidden(reason = "uses toFile")
-        public File(String id, String name, Path path, String contentType) {
+        public File(String id, String name, Path path, String contentType, boolean inline) {
-            super(id, name, contentType);
+            super(id, name, contentType, inline);
             this.path = path;
             this.dataSource = new FileDataSource(path.toFile());
         }
@@ -105,16 +117,16 @@ public abstract class Attachment extends BodyPartSource {
 
         private final byte[] bytes;
 
-        public Bytes(String id, byte[] bytes, String contentType) {
+        public Bytes(String id, byte[] bytes, String contentType, boolean inline) {
-            this(id, id, bytes, contentType);
+            this(id, id, bytes, contentType, inline);
         }
 
-        public Bytes(String id, String name, byte[] bytes) {
+        public Bytes(String id, String name, byte[] bytes, boolean inline) {
-            this(id, name, bytes, fileTypeMap.getContentType(name));
+            this(id, name, bytes, fileTypeMap.getContentType(name), inline);
         }
 
-        public Bytes(String id, String name, byte[] bytes, String contentType) {
+        public Bytes(String id, String name, byte[] bytes, String contentType, boolean inline) {
-            super(id, name, contentType);
+            super(id, name, contentType, inline);
             this.bytes = bytes;
         }
 
@@ -134,6 +146,68 @@ public abstract class Attachment extends BodyPartSource {
         }
     }
 
+    public static class Stream extends Attachment {
+
+        static final String TYPE = "stream";
+
+        private final Provider<InputStream> source;
+
+        public Stream(String id, String name, boolean inline, Provider<InputStream> source) {
+            this(id, name, fileTypeMap.getContentType(name), inline, source);
+        }
+
+        public Stream(String id, String name, String contentType, boolean inline, Provider<InputStream> source) {
+            super(id, name, contentType, inline);
+            this.source = source;
+        }
+
+        @Override
+        public String type() {
+            return TYPE;
+        }
+
+        @Override
+        protected void writeTo(MimeBodyPart part) throws MessagingException {
+            DataSource ds = new StreamDataSource(name, contentType, source);
+            DataHandler dh = new DataHandler(ds);
+            part.setDataHandler(dh);
+        }
+
+        static class StreamDataSource implements DataSource {
+
+            private final String name;
+            private final String contentType;
+            private final Provider<InputStream> source;
+
+            public StreamDataSource(String name, String contentType, Provider<InputStream> source) {
+                this.name = name;
+                this.contentType = contentType;
+                this.source = source;
+            }
+
+            @Override
+            public InputStream getInputStream() throws IOException {
+                return source.get();
+            }
+
+            @Override
+            public OutputStream getOutputStream() throws IOException {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public String getContentType() {
+                return contentType;
+            }
+
+            @Override
+            public String getName() {
+                return name;
+            }
+        }
+
+    }
+
     public static class XContent extends Bytes {
 
         protected XContent(String id, ToXContent content, XContentType type) {
@@ -141,7 +215,7 @@ public abstract class Attachment extends BodyPartSource {
         }
 
         protected XContent(String id, String name, ToXContent content, XContentType type) {
-            super(id, name, bytes(name, content, type), mimeType(type));
+            super(id, name, bytes(name, content, type), mimeType(type), false);
         }
 
         static String mimeType(XContentType type) {

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Email.java

@@ -49,11 +49,10 @@ public class Email implements ToXContent {
     final String textBody;
     final String htmlBody;
     final Map<String, Attachment> attachments;
-    final Map<String, Inline> inlines;
 
     public Email(String id, Address from, AddressList replyTo, Priority priority, DateTime sentDate,
                  AddressList to, AddressList cc, AddressList bcc, String subject, String textBody, String htmlBody,
-                 Map<String, Attachment> attachments, Map<String, Inline> inlines) {
+                 Map<String, Attachment> attachments) {
 
         this.id = id;
         this.from = from;
@@ -67,7 +66,6 @@ public class Email implements ToXContent {
         this.textBody = textBody;
         this.htmlBody = htmlBody;
         this.attachments = attachments;
-        this.inlines = inlines;
     }
 
     public String id() {
@@ -118,10 +116,6 @@ public class Email implements ToXContent {
         return attachments;
     }
 
-    public Map<String, Inline> inlines() {
-        return inlines;
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -249,7 +243,6 @@ public class Email implements ToXContent {
         private String textBody;
         private String htmlBody;
         private Map<String, Attachment> attachments = new HashMap<>();
-        private Map<String, Inline> inlines = new HashMap<>();
 
         private Builder() {
         }
@@ -267,7 +260,6 @@ public class Email implements ToXContent {
             textBody = email.textBody;
             htmlBody = email.htmlBody;
             attachments.putAll(email.attachments);
-            inlines.putAll(email.inlines);
             return this;
         }
 
@@ -358,14 +350,6 @@ public class Email implements ToXContent {
             return this;
         }
 
-        public Builder inline(Inline inline) {
-            if (inlines == null) {
-                throw new IllegalStateException("Email has already been built!");
-            }
-            inlines.put(inline.id(), inline);
-            return this;
-        }
-
         /**
          * Build the email. Note that adding items to attachments or inlines
          * after this is called is incorrect.
@@ -373,9 +357,8 @@ public class Email implements ToXContent {
         public Email build() {
             assert id != null : "email id should not be null (should be set to the watch id";
             Email email = new Email(id, from, replyTo, priority, sentDate, to, cc, bcc, subject, textBody, htmlBody,
-                    unmodifiableMap(attachments), unmodifiableMap(inlines));
+                    unmodifiableMap(attachments));
             attachments = null;
-            inlines = null;
             return email;
         }
 

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Inline.java

@@ -1,189 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.notification.email;
-
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.notification.email.support.BodyPartSource;
-
-import javax.activation.DataHandler;
-import javax.activation.DataSource;
-import javax.activation.FileDataSource;
-import javax.mail.MessagingException;
-import javax.mail.Part;
-import javax.mail.internet.MimeBodyPart;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.file.Path;
-
-/**
- *
- */
-public abstract class Inline extends BodyPartSource {
-
-    protected Inline(String id, String name, String contentType) {
-        super(id, name, contentType);
-    }
-
-    public abstract String type();
-
-    @Override
-    public final MimeBodyPart bodyPart() throws MessagingException {
-        MimeBodyPart part = new MimeBodyPart();
-        part.setDisposition(Part.INLINE);
-        part.setContentID(id);
-        part.setFileName(name);
-        writeTo(part);
-        return part;
-    }
-
-    /**
-     * intentionally not emitting path as it may come as an information leak
-     */
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return builder.startObject()
-                .field("type", type())
-                .field("id", id)
-                .field("name", name)
-                .field("content_type", contentType)
-                .endObject();
-    }
-
-    protected abstract void writeTo(MimeBodyPart part) throws MessagingException;
-
-    public static class File extends Inline {
-
-        static final String TYPE = "file";
-
-        private final Path path;
-        private DataSource dataSource;
-
-        public File(String id, Path path) {
-            this(id, path.getFileName().toString(), path);
-        }
-
-        @SuppressForbidden(reason = "uses toFile")
-        public File(String id, String name, Path path) {
-            this(id, name, path, fileTypeMap.getContentType(path.toFile()));
-        }
-
-        @SuppressForbidden(reason = "uses toFile")
-        public File(String id, String name, Path path, String contentType) {
-            super(id, name, contentType);
-            this.path = path;
-            this.dataSource = new FileDataSource(path.toFile());
-        }
-
-        public Path path() {
-            return path;
-        }
-
-        public String type() {
-            return TYPE;
-        }
-
-        @Override
-        public void writeTo(MimeBodyPart part) throws MessagingException {
-            part.setDataHandler(new DataHandler(dataSource, contentType));
-        }
-    }
-
-    public static class Stream extends Inline {
-
-        static final String TYPE = "stream";
-
-        private final Provider<InputStream> source;
-
-        public Stream(String id, String name, Provider<InputStream> source) {
-            this(id, name, fileTypeMap.getContentType(name), source);
-        }
-
-        public Stream(String id, String name, String contentType, Provider<InputStream> source) {
-            super(id, name, contentType);
-            this.source = source;
-        }
-
-        @Override
-        public String type() {
-            return TYPE;
-        }
-
-        @Override
-        protected void writeTo(MimeBodyPart part) throws MessagingException {
-            DataSource ds = new StreamDataSource(name, contentType, source);
-            DataHandler dh = new DataHandler(ds);
-            part.setDataHandler(dh);
-        }
-
-        static class StreamDataSource implements DataSource {
-
-            private final String name;
-            private final String contentType;
-            private final Provider<InputStream> source;
-
-            public StreamDataSource(String name, String contentType, Provider<InputStream> source) {
-                this.name = name;
-                this.contentType = contentType;
-                this.source = source;
-            }
-
-            @Override
-            public InputStream getInputStream() throws IOException {
-                return source.get();
-            }
-
-            @Override
-            public OutputStream getOutputStream() throws IOException {
-                throw new UnsupportedOperationException();
-            }
-
-            @Override
-            public String getContentType() {
-                return contentType;
-            }
-
-            @Override
-            public String getName() {
-                return name;
-            }
-        }
-    }
-
-    public static class Bytes extends Stream {
-
-        public Bytes(String id, String name, String contentType, byte[] bytes) {
-            super(id, name, contentType, new BytesStreamProvider(bytes));
-        }
-
-        public Bytes(String id, String name, String contentType, BytesReference bytes) {
-            super(id, name, contentType, new BytesStreamProvider(bytes));
-        }
-
-        static class BytesStreamProvider implements Provider<InputStream> {
-
-            private final BytesReference bytes;
-
-            public BytesStreamProvider(byte[] bytes) {
-                this(new BytesArray(bytes));
-            }
-
-            public BytesStreamProvider(BytesReference bytes) {
-                this.bytes = bytes;
-            }
-
-            @Override
-            public InputStream get() {
-                return new ByteArrayInputStream(bytes.array(), bytes.arrayOffset(), bytes.length());
-            }
-        }
-    }
-}

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Profile.java

@@ -92,15 +92,13 @@ public enum Profile implements ToXContent {
                 alternative.addBodyPart(html);
             }
 
-            if (!email.inlines.isEmpty()) {
-                for (Inline inline : email.inlines.values()) {
-                    related.addBodyPart(inline.bodyPart());
-                }
-            }
-
             if (!email.attachments.isEmpty()) {
                 for (Attachment attachment : email.attachments.values()) {
-                    mixed.addBodyPart(attachment.bodyPart());
+                    if (attachment.isInline()) {
+                        related.addBodyPart(attachment.bodyPart());
+                    } else {
+                        mixed.addBodyPart(attachment.bodyPart());
+                    }
                 }
             }
 

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/DataAttachment.java

@@ -54,10 +54,16 @@ public class DataAttachment implements EmailAttachmentParser.EmailAttachment {
         return Objects.hash(id, dataAttachment);
     }
 
+    @Override
     public String id() {
         return id;
     }
 
+    @Override
+    public boolean inline() {
+        return false;
+    }
+
     public static Builder builder(String id) {
         return new Builder(id);
     }

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParser.java

@@ -30,6 +30,14 @@ public interface EmailAttachmentParser<T extends EmailAttachmentParser.EmailAtta
          * @return The id of this attachment
          */
         String id();
+
+        /**
+         * Allows the attachment to decide of it should be of disposition type attachment or inline, which is important
+         * for being able to display inside of desktop email clients
+         *
+         * @return a boolean flagging this attachment as being inline
+         */
+        boolean inline();
     }
 
     /**

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/HttpEmailAttachementParser.java

@@ -14,15 +14,15 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
-import org.elasticsearch.xpack.watcher.support.Variables;
 import org.elasticsearch.xpack.common.http.HttpClient;
 import org.elasticsearch.xpack.common.http.HttpRequest;
 import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.common.http.HttpResponse;
 import org.elasticsearch.xpack.common.text.TextTemplateEngine;
-import org.elasticsearch.xpack.watcher.watch.Payload;
 import org.elasticsearch.xpack.notification.email.Attachment;
+import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
+import org.elasticsearch.xpack.watcher.support.Variables;
+import org.elasticsearch.xpack.watcher.watch.Payload;
 
 import java.io.IOException;
 import java.util.Map;
@@ -30,6 +30,7 @@ import java.util.Map;
 public class HttpEmailAttachementParser implements EmailAttachmentParser<HttpRequestAttachment> {
 
     public interface Fields {
+        ParseField INLINE = new ParseField("inline");
         ParseField REQUEST = new ParseField("request");
         ParseField CONTENT_TYPE = new ParseField("content_type");
     }
@@ -56,6 +57,7 @@ public class HttpEmailAttachementParser implements EmailAttachmentParser<HttpReq
 
     @Override
     public HttpRequestAttachment parse(String id, XContentParser parser) throws IOException {
+        boolean inline = false;
         String contentType = null;
         HttpRequestTemplate requestTemplate = null;
 
@@ -66,6 +68,8 @@ public class HttpEmailAttachementParser implements EmailAttachmentParser<HttpReq
                 currentFieldName = parser.currentName();
             } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.CONTENT_TYPE)) {
                 contentType = parser.text();
+            } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.INLINE)) {
+                inline = parser.booleanValue();
             } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.REQUEST)) {
                 requestTemplate = requestTemplateParser.parse(parser);
             } else {
@@ -75,7 +79,7 @@ public class HttpEmailAttachementParser implements EmailAttachmentParser<HttpReq
         }
 
         if (requestTemplate != null) {
-            return new HttpRequestAttachment(id, requestTemplate, contentType);
+            return new HttpRequestAttachment(id, requestTemplate, inline, contentType);
         }
 
         throw new ElasticsearchParseException("Could not parse http request attachment");
@@ -94,7 +98,7 @@ public class HttpEmailAttachementParser implements EmailAttachmentParser<HttpReq
                 if (response.hasContent()) {
                     String contentType = attachment.getContentType();
                     String attachmentContentType = Strings.hasLength(contentType) ? contentType : response.contentType();
-                    return new Attachment.Bytes(attachment.id(), response.body().toBytes(), attachmentContentType);
+                    return new Attachment.Bytes(attachment.id(), response.body().toBytes(), attachmentContentType, attachment.inline());
                 } else {
                     logger.error("Empty response body: [host[{}], port[{}], method[{}], path[{}]: response status [{}]", httpRequest.host(),
                             httpRequest.port(), httpRequest.method(), httpRequest.path(), response.status());

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/HttpRequestAttachment.java

@@ -16,12 +16,14 @@ import java.util.Objects;
 public class HttpRequestAttachment implements EmailAttachmentParser.EmailAttachment {
 
     private final HttpRequestTemplate requestTemplate;
+    private boolean inline;
     private final String contentType;
     private final String id;
 
-    public HttpRequestAttachment(String id, HttpRequestTemplate requestTemplate, @Nullable String contentType) {
+    public HttpRequestAttachment(String id, HttpRequestTemplate requestTemplate, boolean inline, @Nullable String contentType) {
         this.id = id;
         this.requestTemplate = requestTemplate;
+        this.inline = inline;
         this.contentType = contentType;
     }
 
@@ -38,6 +40,11 @@ public class HttpRequestAttachment implements EmailAttachmentParser.EmailAttachm
         return id;
     }
 
+    @Override
+    public boolean inline() {
+        return inline;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(id)
@@ -46,6 +53,9 @@ public class HttpRequestAttachment implements EmailAttachmentParser.EmailAttachm
         if (Strings.hasLength(contentType)) {
             builder.field(HttpEmailAttachementParser.Fields.CONTENT_TYPE.getPreferredName(), contentType);
         }
+        if (inline) {
+            builder.field(HttpEmailAttachementParser.Fields.INLINE.getPreferredName(), inline);
+        }
         return builder.endObject().endObject();
     }
 
@@ -65,12 +75,12 @@ public class HttpRequestAttachment implements EmailAttachmentParser.EmailAttachm
 
         HttpRequestAttachment otherDataAttachment = (HttpRequestAttachment) o;
         return Objects.equals(id, otherDataAttachment.id) && Objects.equals(requestTemplate, otherDataAttachment.requestTemplate)
-                && Objects.equals(contentType, otherDataAttachment.contentType);
+                && Objects.equals(contentType, otherDataAttachment.contentType) && Objects.equals(inline, otherDataAttachment.inline);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, requestTemplate, contentType);
+        return Objects.hash(id, requestTemplate, contentType, inline);
     }
 
     public static class Builder {
@@ -78,6 +88,7 @@ public class HttpRequestAttachment implements EmailAttachmentParser.EmailAttachm
         private String id;
         private HttpRequestTemplate httpRequestTemplate;
         private String contentType;
+        private boolean inline = false;
 
         private Builder(String id) {
             this.id = id;
@@ -93,8 +104,13 @@ public class HttpRequestAttachment implements EmailAttachmentParser.EmailAttachm
             return this;
         }
 
+        public Builder inline(boolean inline) {
+            this.inline = inline;
+            return this;
+        }
+
         public HttpRequestAttachment build() {
-            return new HttpRequestAttachment(id, httpRequestTemplate, contentType);
+            return new HttpRequestAttachment(id, httpRequestTemplate, inline, contentType);
         }
 
     }

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/common/http/HttpResponseTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.common.http;
 
-import com.google.common.collect.Lists;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -13,9 +12,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.common.http.HttpResponse;
 
 import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -29,8 +28,8 @@ import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 
 /**
  *
@@ -122,9 +121,9 @@ public class HttpResponseTests extends ESTestCase {
         Map<String, Object> responseHeaders = (Map<String, Object>) responseMap.get("headers");
 
         assertThat(responseHeaders, not(hasKey("es.index")));
-        assertThat(responseHeaders, hasEntry("es_index", Lists.newArrayList("value")));
+        assertThat(responseHeaders, hasEntry("es_index", Collections.singletonList("value")));
 
         assertThat(responseHeaders, not(hasKey("es.index.2")));
-        assertThat(responseHeaders, hasEntry("es_index_2", Lists.newArrayList("value")));
+        assertThat(responseHeaders, hasEntry("es_index_2", Collections.singletonList("value")));
     }
 }

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/EmailTests.java

@@ -41,9 +41,8 @@ public class EmailTests extends ESTestCase {
         String textBody = randomFrom("Random Body", "", null);
         String htmlBody = randomFrom("<hr /><b>BODY</b><hr />", "", null);
         Map<String, Attachment> attachments = null;
-        Map<String, Inline> inlines = null;
 
-        Email email = new Email(id, from, replyTo, priority, sentDate, to, cc, bcc, subject, textBody, htmlBody, attachments, inlines);
+        Email email = new Email(id, from, replyTo, priority, sentDate, to, cc, bcc, subject, textBody, htmlBody, attachments);
 
         XContentBuilder builder = XContentFactory.jsonBuilder();
         email.toXContent(builder, ToXContent.EMPTY_PARAMS);

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ManualPublicSmtpServersTester.java

@@ -6,14 +6,13 @@
 package org.elasticsearch.xpack.notification.email;
 
 import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.common.secret.SecretService;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.Locale;
 
@@ -26,13 +25,13 @@ public class ManualPublicSmtpServersTester {
 
         public static void main(String[] args) throws Exception {
             test(Profile.GMAIL, Settings.builder()
-                    .put("xpack.notification.email.service.account.gmail.smtp.auth", true)
+                    .put("xpack.notification.email.account.gmail.smtp.auth", true)
-                    .put("xpack.notification.email.service.account.gmail.smtp.starttls.enable", true)
+                    .put("xpack.notification.email.account.gmail.smtp.starttls.enable", true)
-                    .put("xpack.notification.email.service.account.gmail.smtp.host", "smtp.gmail.com")
+                    .put("xpack.notification.email.account.gmail.smtp.host", "smtp.gmail.com")
-                    .put("xpack.notification.email.service.account.gmail.smtp.port", 587)
+                    .put("xpack.notification.email.account.gmail.smtp.port", 587)
-                    .put("xpack.notification.email.service.account.gmail.smtp.user", terminal.readText("username: "))
+                    .put("xpack.notification.email.account.gmail.smtp.user", terminal.readText("username: "))
-                    .put("xpack.notification.email.service.account.gmail.smtp.password", new String(terminal.readSecret("password: ")))
+                    .put("xpack.notification.email.account.gmail.smtp.password", new String(terminal.readSecret("password: ")))
-                    .put("xpack.notification.email.service.account.gmail.email_defaults.to", terminal.readText("to: "))
+                    .put("xpack.notification.email.account.gmail.email_defaults.to", terminal.readText("to: "))
             );
         }
     }
@@ -41,13 +40,13 @@ public class ManualPublicSmtpServersTester {
 
         public static void main(String[] args) throws Exception {
             test(Profile.STANDARD, Settings.builder()
-                    .put("xpack.notification.email.service.account.outlook.smtp.auth", true)
+                    .put("xpack.notification.email.account.outlook.smtp.auth", true)
-                    .put("xpack.notification.email.service.account.outlook.smtp.starttls.enable", true)
+                    .put("xpack.notification.email.account.outlook.smtp.starttls.enable", true)
-                    .put("xpack.notification.email.service.account.outlook.smtp.host", "smtp-mail.outlook.com")
+                    .put("xpack.notification.email.account.outlook.smtp.host", "smtp-mail.outlook.com")
-                    .put("xpack.notification.email.service.account.outlook.smtp.port", 587)
+                    .put("xpack.notification.email.account.outlook.smtp.port", 587)
-                    .put("xpack.notification.email.service.account.outlook.smtp.user", "elastic.user@outlook.com")
+                    .put("xpack.notification.email.account.outlook.smtp.user", "elastic.user@outlook.com")
-                    .put("xpack.notification.email.service.account.outlook.smtp.password", "fantastic42")
+                    .put("xpack.notification.email.account.outlook.smtp.password", "fantastic42")
-                    .put("xpack.notification.email.service.account.outlook.email_defaults.to", "elastic.user@outlook.com")
+                    .put("xpack.notification.email.account.outlook.email_defaults.to", "elastic.user@outlook.com")
                     .put()
             );
         }
@@ -57,15 +56,15 @@ public class ManualPublicSmtpServersTester {
 
         public static void main(String[] args) throws Exception {
             test(Profile.STANDARD, Settings.builder()
-                            .put("xpack.notification.email.service.account.yahoo.smtp.starttls.enable", true)
+                            .put("xpack.notification.email.account.yahoo.smtp.starttls.enable", true)
-                            .put("xpack.notification.email.service.account.yahoo.smtp.auth", true)
+                            .put("xpack.notification.email.account.yahoo.smtp.auth", true)
-                            .put("xpack.notification.email.service.account.yahoo.smtp.host", "smtp.mail.yahoo.com")
+                            .put("xpack.notification.email.account.yahoo.smtp.host", "smtp.mail.yahoo.com")
-                            .put("xpack.notification.email.service.account.yahoo.smtp.port", 587)
+                            .put("xpack.notification.email.account.yahoo.smtp.port", 587)
-                            .put("xpack.notification.email.service.account.yahoo.smtp.user", "elastic.user@yahoo.com")
+                            .put("xpack.notification.email.account.yahoo.smtp.user", "elastic.user@yahoo.com")
-                            .put("xpack.notification.email.service.account.yahoo.smtp.password", "fantastic42")
+                            .put("xpack.notification.email.account.yahoo.smtp.password", "fantastic42")
                             // note: from must be set to the same authenticated user account
-                            .put("xpack.notification.email.service.account.yahoo.email_defaults.from", "elastic.user@yahoo.com")
+                            .put("xpack.notification.email.account.yahoo.email_defaults.from", "elastic.user@yahoo.com")
-                            .put("xpack.notification.email.service.account.yahoo.email_defaults.to", "elastic.user@yahoo.com")
+                            .put("xpack.notification.email.account.yahoo.email_defaults.to", "elastic.user@yahoo.com")
             );
         }
     }
@@ -75,42 +74,43 @@ public class ManualPublicSmtpServersTester {
 
         public static void main(String[] args) throws Exception {
             test(Profile.STANDARD, Settings.builder()
-                            .put("xpack.notification.email.service.account.ses.smtp.auth", true)
+                            .put("xpack.notification.email.account.ses.smtp.auth", true)
-                            .put("xpack.notification.email.service.account.ses.smtp.starttls.enable", true)
+                            .put("xpack.notification.email.account.ses.smtp.starttls.enable", true)
-                            .put("xpack.notification.email.service.account.ses.smtp.starttls.required", true)
+                            .put("xpack.notification.email.account.ses.smtp.starttls.required", true)
-                            .put("xpack.notification.email.service.account.ses.smtp.host", "email-smtp.us-east-1.amazonaws.com")
+                            .put("xpack.notification.email.account.ses.smtp.host", "email-smtp.us-east-1.amazonaws.com")
-                            .put("xpack.notification.email.service.account.ses.smtp.port", 587)
+                            .put("xpack.notification.email.account.ses.smtp.port", 587)
-                            .put("xpack.notification.email.service.account.ses.smtp.user", terminal.readText("user: "))
+                            .put("xpack.notification.email.account.ses.smtp.user", terminal.readText("user: "))
-                            .put("xpack.notification.email.service.account.ses.email_defaults.from", "dummy.user@elasticsearch.com")
+                            .put("xpack.notification.email.account.ses.email_defaults.from", "dummy.user@elasticsearch.com")
-                            .put("xpack.notification.email.service.account.ses.email_defaults.to", terminal.readText("to: "))
+                            .put("xpack.notification.email.account.ses.email_defaults.to", terminal.readText("to: "))
-                            .put("xpack.notification.email.service.account.ses.smtp.password",
+                            .put("xpack.notification.email.account.ses.smtp.password",
                                     new String(terminal.readSecret("password: ")))
             );
         }
     }
 
-    static void test(Profile profile, Settings.Builder builder) throws Exception {
+    static void test(Profile profile, Settings.Builder settingsBuilder) throws Exception {
-        InternalEmailService service = startEmailService(builder);
+        String path = "/org/elasticsearch/xpack/watcher/actions/email/service/logo.png";
+        if (InternalEmailServiceTests.class.getResourceAsStream(path) == null) {
+            throw new ElasticsearchException("Could not find logo at path {}", path);
+        }
+
+        InternalEmailService service = startEmailService(settingsBuilder);
         try {
 
-            ToXContent content = new ToXContent() {
+            ToXContent content = (xContentBuilder, params) -> xContentBuilder.startObject()
-                @Override
+                    .field("key1", "value1")
-                public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                    .field("key2", "value2")
-                    return builder.startObject()
+                    .field("key3", "value3")
-                            .field("key1", "value1")
+                    .endObject();
-                            .field("key2", "value2")
-                            .field("key3", "value3")
-                            .endObject();
-                }
-            };
 
             Email email = Email.builder()
                     .id("_id")
                     .subject("_subject")
                     .textBody("_text_body")
-                    .htmlBody("<b>html body</b><p/><p/><img src=\"cid:logo\"/>")
+                    .htmlBody("<b>html body</b><p/><p/><img src=\"cid:logo.png\"/>")
                     .attach(new Attachment.XContent.Yaml("test.yml", content))
-                    .inline(new Inline.Stream("logo", "logo.jpg", () -> InternalEmailServiceTests.class.getResourceAsStream("logo.png")))
+                    .attach(new Attachment.Stream("logo.png", "logo.png", true,
+                            () -> InternalEmailServiceTests.class.getResourceAsStream(path)))
                     .build();
 
             EmailService.EmailSent sent = service.send(email, null, profile);

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ProfileTests.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.notification.email;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.common.secret.SecretService;
+
+import javax.mail.BodyPart;
+import javax.mail.Part;
+import javax.mail.Session;
+import javax.mail.internet.MimeMessage;
+import javax.mail.internet.MimeMultipart;
+
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class ProfileTests extends ESTestCase {
+
+    public void testThatInlineAttachmentsAreCreated() throws Exception {
+        String path = "/org/elasticsearch/xpack/watcher/actions/email/service/logo.png";
+        Attachment attachment = new Attachment.Stream("inline.png", "inline.png", true,
+                () -> InternalEmailServiceTests.class.getResourceAsStream(path));
+
+        Email email = Email.builder()
+                .id("foo")
+                .from("foo@example.org")
+                .to("bar@example.org")
+                .subject(randomAsciiOfLength(10))
+                .attach(attachment)
+                .build();
+
+        Settings settings = Settings.builder()
+                .put("default_account", "foo")
+                .put("account.foo.smtp.host", "_host")
+                .build();
+
+        Accounts accounts = new Accounts(settings, SecretService.Insecure.INSTANCE, logger);
+        Session session = accounts.account("foo").getConfig().createSession();
+        MimeMessage mimeMessage = Profile.STANDARD.toMimeMessage(email, session);
+
+        Object content = ((MimeMultipart) mimeMessage.getContent()).getBodyPart(0).getContent();
+        assertThat(content, instanceOf(MimeMultipart.class));
+        MimeMultipart multipart = (MimeMultipart) content;
+
+        assertThat(multipart.getCount(), is(2));
+        boolean foundInlineAttachment = false;
+        BodyPart bodyPart = null;
+        for (int i = 0; i < multipart.getCount(); i++) {
+            bodyPart = multipart.getBodyPart(i);
+            if (Part.INLINE.equalsIgnoreCase(bodyPart.getDisposition())) {
+                foundInlineAttachment = true;
+                break;
+            }
+        }
+
+        assertThat("Expected to find an inline attachment in mime message, but didnt", foundInlineAttachment, is(true));
+    }
+}

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParsersTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.notification.email.attachment;
 
-import com.google.common.base.Charsets;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -13,13 +12,14 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.common.http.Scheme;
-import org.elasticsearch.xpack.watcher.watch.Payload;
 import org.elasticsearch.xpack.notification.email.Attachment;
+import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
+import org.elasticsearch.xpack.watcher.watch.Payload;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -93,7 +93,8 @@ public class EmailAttachmentParsersTests extends ESTestCase {
         attachments.add(new DataAttachment("my-name.json", org.elasticsearch.xpack.notification.email.DataAttachment.JSON));
 
         HttpRequestTemplate requestTemplate = HttpRequestTemplate.builder("localhost", 80).scheme(Scheme.HTTP).path("/").build();
-        HttpRequestAttachment httpRequestAttachment = new HttpRequestAttachment("other-id", requestTemplate, null);
+        boolean inline = randomBoolean();
+        HttpRequestAttachment httpRequestAttachment = new HttpRequestAttachment("other-id", requestTemplate, inline, null);
 
         attachments.add(httpRequestAttachment);
         EmailAttachments emailAttachments = new EmailAttachments(attachments);
@@ -105,6 +106,9 @@ public class EmailAttachmentParsersTests extends ESTestCase {
         assertThat(builder.string(), containsString("other-id"));
         assertThat(builder.string(), containsString("localhost"));
         assertThat(builder.string(), containsString("/"));
+        if (inline) {
+            assertThat(builder.string(), containsString("inline"));
+        }
     }
 
     public void testThatTwoAttachmentsWithTheSameIdThrowError() throws Exception {
@@ -161,7 +165,8 @@ public class EmailAttachmentParsersTests extends ESTestCase {
 
         @Override
         public Attachment toAttachment(WatchExecutionContext ctx, Payload payload, TestEmailAttachment attachment) {
-            return new Attachment.Bytes(attachment.id(), attachment.getValue().getBytes(Charsets.UTF_8), "personalContentType");
+            return new Attachment.Bytes(attachment.id(), attachment.getValue().getBytes(StandardCharsets.UTF_8),
+                                        "personalContentType", false);
         }
     }
 
@@ -193,6 +198,11 @@ public class EmailAttachmentParsersTests extends ESTestCase {
             return id;
         }
 
+        @Override
+        public boolean inline() {
+            return false;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             return builder.startObject(id)

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/HttpEmailAttachementParserTests.java

@@ -37,15 +37,13 @@ import static org.mockito.Mockito.when;
 
 public class HttpEmailAttachementParserTests extends ESTestCase {
 
-    private SecretService.Insecure secretService;
-    private HttpAuthRegistry authRegistry;
     private HttpRequestTemplate.Parser httpRequestTemplateParser;
     private HttpClient httpClient;
 
     @Before
     public void init() throws Exception {
-        secretService = SecretService.Insecure.INSTANCE;
+        SecretService.Insecure secretService = SecretService.Insecure.INSTANCE;
-        authRegistry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(secretService)));
+        HttpAuthRegistry authRegistry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(secretService)));
         httpRequestTemplateParser = new HttpRequestTemplate.Parser(authRegistry);
         httpClient = mock(HttpClient.class);
 
@@ -77,9 +75,12 @@ public class HttpEmailAttachementParserTests extends ESTestCase {
         if (configureContentType) {
             builder.field("content_type", "application/foo");
         }
+        boolean isInline = randomBoolean();
+        if (isInline) {
+            builder.field("inline", true);
+        }
         builder.endObject().endObject().endObject();
         XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes());
-        logger.info("JSON: {}", builder.string());
 
         EmailAttachments emailAttachments = emailAttachmentsParser.parse(parser);
         assertThat(emailAttachments.getAttachments(), hasSize(1));
@@ -89,6 +90,7 @@ public class HttpEmailAttachementParserTests extends ESTestCase {
         attachments.get(0).toXContent(toXcontentBuilder, ToXContent.EMPTY_PARAMS);
         toXcontentBuilder.endObject();
         assertThat(toXcontentBuilder.string(), is(builder.string()));
-    }
 
+        assertThat(attachments.get(0).inline(), is(isInline));
+    }
 }

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java

@@ -5,26 +5,10 @@
  */
 package org.elasticsearch.xpack.test.rest;
 
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -33,15 +17,25 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.license.plugin.TestUtils;
-import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.client.RestTestResponse;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
+import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
+import org.elasticsearch.xpack.security.authc.support.SecuredString;
+import org.elasticsearch.xpack.security.authz.store.ReservedRolesStore;
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-import static org.hamcrest.Matchers.is;
 
 public abstract class XPackRestTestCase extends ESRestTestCase {
 
@@ -56,46 +50,16 @@ public abstract class XPackRestTestCase extends ESRestTestCase {
         return ESRestTestCase.createParameters(0, 1);
     }
 
-    @Before
-    public void startWatcher() throws Exception {
-        try (CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http",
-                                                  null,
-                                                  url.getHost(),
-                                                  url.getPort(),
-                                                  "/_xpack/watcher/_start", null, null));
-            request.addHeader("Authorization", BASIC_AUTH_VALUE);
-            try (CloseableHttpResponse response = client.execute(request)) {
-            }
-        }
-    }
-
-    @After
-    public void stopWatcher() throws Exception {
-        try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
-            URL url = getClusterUrls()[0];
-            HttpPut request = new HttpPut(new URI("http",
-                                                  null,
-                                                  url.getHost(),
-                                                  url.getPort(),
-                                                  "/_xpack/watcher/_stop", null, null));
-            request.addHeader("Authorization", BASIC_AUTH_VALUE);
-            try (CloseableHttpResponse response = client.execute(request)) {
-            }
-        }
-    }
-
     @Before
     public void installLicense() throws Exception {
         final XContentBuilder builder = XContentFactory.jsonBuilder();
         TestUtils.generateSignedLicense("trial", TimeValue.timeValueHours(2)).toXContent(builder, ToXContent.EMPTY_PARAMS);
         final BytesReference bytes = builder.bytes();
         try (XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes)) {
-            final List<Map<String, Object>> bodies = Collections.singletonList(Collections.singletonMap("license",
+            final List<Map<String, Object>> bodies = singletonList(singletonMap("license",
                     parser.map()));
-            getAdminExecutionContext().callApi("license.post", Collections.singletonMap("acknowledge", "true"),
+            getAdminExecutionContext().callApi("license.post", singletonMap("acknowledge", "true"),
-                    bodies, Collections.singletonMap("Authorization", BASIC_AUTH_VALUE));
+                    bodies, singletonMap("Authorization", BASIC_AUTH_VALUE));
         }
     }
 
@@ -103,43 +67,21 @@ public abstract class XPackRestTestCase extends ESRestTestCase {
     public void clearUsersAndRoles() throws Exception {
         // we cannot delete the .security index from a rest test since we aren't the internal user, lets wipe the data
         // TODO remove this once the built-in SUPERUSER role is added that can delete the index and we use the built in admin user here
-        try (CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) {
+        RestTestResponse response = getAdminExecutionContext().callApi("xpack.security.get_user", emptyMap(), emptyList(), emptyMap());
-            final URL url = getClusterUrls()[0];
+        @SuppressWarnings("unchecked")
-            HttpGet getUsersRequest = new HttpGet(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/security/user", null, null));
+        Map<String, Object> users = (Map<String, Object>) response.getBody();
-            getUsersRequest.addHeader("Authorization", BASIC_AUTH_VALUE);
+        for (String user: users.keySet()) {
-            try (CloseableHttpResponse closeableHttpResponse = client.execute(getUsersRequest)) {
+            if (ReservedRealm.isReserved(user) == false) {
-                assertThat(closeableHttpResponse.getStatusLine().getStatusCode(), is(200));
+                getAdminExecutionContext().callApi("xpack.security.delete_user", singletonMap("username", user), emptyList(), emptyMap());
-                String response = Streams.copyToString(
-                        new InputStreamReader(closeableHttpResponse.getEntity().getContent(), StandardCharsets.UTF_8));
-                Map<String, Object> responseMap = XContentFactory.xContent(response).createParser(response).map();
-
-                // in the structure of this API, the users are the keyset
-                for (String user : responseMap.keySet()) {
-                    HttpDelete delete = new HttpDelete(new URI("http", null, url.getHost(), url.getPort(),
-                            "/_xpack/security/user/" + user, null, null));
-                    delete.addHeader("Authorization", BASIC_AUTH_VALUE);
-                    try (CloseableHttpResponse deleteResponse = client.execute(delete)) {
-                    }
-                }
             }
+        }
 
-            HttpGet getRolesRequest = new HttpGet(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/security/role",
+        response = getAdminExecutionContext().callApi("xpack.security.get_role", emptyMap(), emptyList(), emptyMap());
-                    null, null));
+        @SuppressWarnings("unchecked")
-            getRolesRequest.addHeader("Authorization", BASIC_AUTH_VALUE);
+        Map<String, Object> roles = (Map<String, Object>) response.getBody();
-            try (CloseableHttpResponse closeableHttpResponse = client.execute(getRolesRequest)) {
+        for (String role: roles.keySet()) {
-                assertThat(closeableHttpResponse.getStatusLine().getStatusCode(), is(200));
+            if (ReservedRolesStore.isReserved(role) == false) {
-                String response = Streams.copyToString(
+                getAdminExecutionContext().callApi("xpack.security.delete_role", singletonMap("name", role), emptyList(), emptyMap());
-                        new InputStreamReader(closeableHttpResponse.getEntity().getContent(), StandardCharsets.UTF_8));
-                Map<String, Object> responseMap = XContentFactory.xContent(response).createParser(response).map();
-
-                // in the structure of this API, the users are the keyset
-                for (String role : responseMap.keySet()) {
-                    HttpDelete delete = new HttpDelete(new URI("http", null, url.getHost(), url.getPort(),
-                            "/_xpack/security/role/" + role, null, null));
-                    delete.addHeader("Authorization", BASIC_AUTH_VALUE);
-                    try (CloseableHttpResponse deleteResponse = client.execute(delete)) {
-                    }
-                }
             }
         }
     }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java

@@ -31,7 +31,6 @@ import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor;
 import org.elasticsearch.xpack.watcher.history.HistoryModule;
 import org.elasticsearch.xpack.watcher.history.HistoryStore;
 import org.elasticsearch.xpack.watcher.input.InputModule;
-import org.elasticsearch.xpack.watcher.input.chain.ChainInputFactory;
 import org.elasticsearch.xpack.watcher.rest.action.RestAckWatchAction;
 import org.elasticsearch.xpack.watcher.rest.action.RestActivateWatchAction;
 import org.elasticsearch.xpack.watcher.rest.action.RestDeleteWatchAction;
@@ -266,5 +265,5 @@ public class Watcher {
                 "[.watcher-history-YYYY.MM.dd] are allowed to be created", value);
     }
 
-    
+
 }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/index/ExecutableIndexAction.java

@@ -15,7 +15,6 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.xpack.watcher.actions.Action;
 import org.elasticsearch.xpack.watcher.actions.ExecutableAction;
 import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
@@ -24,6 +23,7 @@ import org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
 import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.watcher.watch.Payload;
+import org.joda.time.DateTime;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -63,19 +63,9 @@ public class ExecutableIndexAction extends ExecutableAction<IndexAction> {
         }
 
         IndexRequest indexRequest = new IndexRequest();
-
         indexRequest.index(action.index);
         indexRequest.type(action.docType);
-
+        data = addTimestampToDocument(data, ctx.executionTime());
-        if (action.executionTimeField != null && !TimestampFieldMapper.NAME.equals(action.executionTimeField)) {
-            if (!(data instanceof HashMap)) {
-                data = new HashMap<>(data); // ensuring mutability
-            }
-            data.put(action.executionTimeField, WatcherDateTimeUtils.formatDate(ctx.executionTime()));
-        } else {
-            indexRequest.timestamp(WatcherDateTimeUtils.formatDate(ctx.executionTime()));
-        }
-
         indexRequest.source(jsonBuilder().prettyPrint().map(data));
 
         if (ctx.simulateAction(actionId)) {
@@ -100,14 +90,7 @@ public class ExecutableIndexAction extends ExecutableAction<IndexAction> {
             IndexRequest indexRequest = new IndexRequest();
             indexRequest.index(action.index);
             indexRequest.type(action.docType);
-            if (action.executionTimeField != null && !TimestampFieldMapper.NAME.equals(action.executionTimeField)) {
+            doc = addTimestampToDocument(doc, ctx.executionTime());
-                if (!(doc instanceof HashMap)) {
-                    doc = new HashMap<>(doc); // ensuring mutability
-                }
-                doc.put(action.executionTimeField, WatcherDateTimeUtils.formatDate(ctx.executionTime()));
-            } else {
-                indexRequest.timestamp(WatcherDateTimeUtils.formatDate(ctx.executionTime()));
-            }
             indexRequest.source(jsonBuilder().prettyPrint().map(doc));
             bulkRequest.add(indexRequest);
         }
@@ -121,6 +104,16 @@ public class ExecutableIndexAction extends ExecutableAction<IndexAction> {
         return new IndexAction.Result.Success(new XContentSource(jsonBuilder.bytes(), XContentType.JSON));
     }
 
+    private Map<String, Object> addTimestampToDocument(Map<String, Object> data, DateTime executionTime) {
+        if (action.executionTimeField != null) {
+            if (!(data instanceof HashMap)) {
+                data = new HashMap<>(data); // ensuring mutability
+            }
+            data.put(action.executionTimeField, WatcherDateTimeUtils.formatDate(executionTime));
+        }
+        return data;
+    }
+
     static void indexResponseToXContent(XContentBuilder builder, IndexResponse response) throws IOException {
         builder.startObject()
                 .field("created", response.isCreated())

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/InputBuilders.java

@@ -6,14 +6,14 @@
 package org.elasticsearch.xpack.watcher.input;
 
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.input.chain.ChainInput;
 import org.elasticsearch.xpack.watcher.input.http.HttpInput;
 import org.elasticsearch.xpack.watcher.input.none.NoneInput;
 import org.elasticsearch.xpack.watcher.input.search.SearchInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
-import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.watch.Payload;
 
 import java.util.HashMap;
@@ -31,12 +31,12 @@ public final class InputBuilders {
         return NoneInput.builder();
     }
 
-    public static SearchInput.Builder searchInput(SearchRequest request) {
+    public static SearchInput.Builder searchInput(WatcherSearchTemplateRequest request) {
         return SearchInput.builder(request);
     }
 
-    public static SearchInput.Builder searchInput(SearchRequestBuilder builder) {
+    public static SearchInput.Builder searchInput(SearchRequest request) {
-        return searchInput(builder.request());
+        return searchInput(new WatcherSearchTemplateRequest(request));
     }
 
     public static SimpleInput.Builder simpleInput() {

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java

@@ -11,16 +11,15 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.watcher.input.ExecutableInput;
-import org.elasticsearch.xpack.watcher.support.WatcherUtils;
 import org.elasticsearch.xpack.watcher.support.XContentFilterKeysUtils;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
 import org.elasticsearch.xpack.watcher.watch.Payload;
 
 import java.util.Map;
@@ -35,11 +34,14 @@ public class ExecutableSearchInput extends ExecutableInput<SearchInput, SearchIn
     public static final SearchType DEFAULT_SEARCH_TYPE = SearchType.QUERY_THEN_FETCH;
 
     private final WatcherClientProxy client;
+    private final WatcherSearchTemplateService searchTemplateService;
     private final @Nullable TimeValue timeout;
 
-    public ExecutableSearchInput(SearchInput input, ESLogger logger, WatcherClientProxy client, @Nullable TimeValue defaultTimeout) {
+    public ExecutableSearchInput(SearchInput input, ESLogger logger, WatcherClientProxy client,
+                                 WatcherSearchTemplateService searchTemplateService, @Nullable TimeValue defaultTimeout) {
         super(input, logger);
         this.client = client;
+        this.searchTemplateService = searchTemplateService;
         this.timeout = input.getTimeout() != null ? input.getTimeout() : defaultTimeout;
     }
 
@@ -47,7 +49,7 @@ public class ExecutableSearchInput extends ExecutableInput<SearchInput, SearchIn
     public SearchInput.Result execute(WatchExecutionContext ctx, Payload payload) {
         SearchRequest request = null;
         try {
-            request = WatcherUtils.createSearchRequestFromPrototype(input.getSearchRequest(), ctx, payload);
+            request = searchTemplateService.createSearchRequestFromPrototype(input.getRequest(), ctx, payload);
             return doExecute(ctx, request);
         } catch (Exception e) {
             logger.error("failed to execute [{}] input for [{}]", e, SearchInput.TYPE, ctx.watch());
@@ -57,8 +59,7 @@ public class ExecutableSearchInput extends ExecutableInput<SearchInput, SearchIn
 
     SearchInput.Result doExecute(WatchExecutionContext ctx, SearchRequest request) throws Exception {
         if (logger.isTraceEnabled()) {
-            ToXContent source = request.source() != null ? request.source() : request.template();
+            logger.trace("[{}] running query for [{}] [{}]", ctx.id(), ctx.watch().id(), XContentHelper.toString(request.source()));
-            logger.trace("[{}] running query for [{}] [{}]", ctx.id(), ctx.watch().id(), XContentHelper.toString(source));
         }
 
         SearchResponse response = client.search(request, timeout);

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInput.java

@@ -17,9 +17,8 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.xpack.watcher.input.Input;
-import org.elasticsearch.xpack.watcher.support.SearchRequestEquivalence;
 import org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils;
-import org.elasticsearch.xpack.watcher.support.WatcherUtils;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.watch.Payload;
 import org.joda.time.DateTimeZone;
 
@@ -38,14 +37,14 @@ public class SearchInput implements Input {
 
     public static final String TYPE = "search";
 
-    private final SearchRequest searchRequest;
+    private final WatcherSearchTemplateRequest request;
     private final @Nullable Set<String> extractKeys;
     private final @Nullable TimeValue timeout;
     private final @Nullable DateTimeZone dynamicNameTimeZone;
 
-    public SearchInput(SearchRequest searchRequest, @Nullable Set<String> extractKeys,
+    public SearchInput(WatcherSearchTemplateRequest request, @Nullable Set<String> extractKeys,
                        @Nullable TimeValue timeout, @Nullable DateTimeZone dynamicNameTimeZone) {
-        this.searchRequest = searchRequest;
+        this.request = request;
         this.extractKeys = extractKeys;
         this.timeout = timeout;
         this.dynamicNameTimeZone = dynamicNameTimeZone;
@@ -63,7 +62,7 @@ public class SearchInput implements Input {
 
         SearchInput that = (SearchInput) o;
 
-        if (!SearchRequestEquivalence.INSTANCE.equivalent(searchRequest, this.searchRequest)) return false;
+        if (request != null ? !request.equals(that.request) : that.request != null) return false;
         if (extractKeys != null ? !extractKeys.equals(that.extractKeys) : that.extractKeys != null) return false;
         if (timeout != null ? !timeout.equals(that.timeout) : that.timeout != null) return false;
         return !(dynamicNameTimeZone != null ? !dynamicNameTimeZone.equals(that.dynamicNameTimeZone) : that.dynamicNameTimeZone != null);
@@ -71,15 +70,15 @@ public class SearchInput implements Input {
 
     @Override
     public int hashCode() {
-        int result = searchRequest.hashCode();
+        int result = request != null ? request.hashCode() : 0;
         result = 31 * result + (extractKeys != null ? extractKeys.hashCode() : 0);
         result = 31 * result + (timeout != null ? timeout.hashCode() : 0);
         result = 31 * result + (dynamicNameTimeZone != null ? dynamicNameTimeZone.hashCode() : 0);
         return result;
     }
 
-    public SearchRequest getSearchRequest() {
+    public WatcherSearchTemplateRequest getRequest() {
-        return searchRequest;
+        return request;
     }
 
     public Set<String> getExtractKeys() {
@@ -97,8 +96,9 @@ public class SearchInput implements Input {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(Field.REQUEST.getPreferredName());
+        if (request != null) {
-        builder = WatcherUtils.writeSearchRequest(searchRequest, builder, params);
+            builder.field(Field.REQUEST.getPreferredName(), request);
+        }
         if (extractKeys != null) {
             builder.field(Field.EXTRACT.getPreferredName(), extractKeys);
         }
@@ -115,7 +115,7 @@ public class SearchInput implements Input {
     public static SearchInput parse(String watchId, XContentParser parser, QueryParseContext context,
                                     AggregatorParsers aggParsers, Suggesters suggesters)
             throws IOException {
-        SearchRequest request = null;
+        WatcherSearchTemplateRequest request = null;
         Set<String> extract = null;
         TimeValue timeout = null;
         DateTimeZone dynamicNameTimeZone = null;
@@ -127,7 +127,7 @@ public class SearchInput implements Input {
                 currentFieldName = parser.currentName();
             } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Field.REQUEST)) {
                 try {
-                    request = WatcherUtils.readSearchRequest(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context,
+                    request = WatcherSearchTemplateRequest.fromXContent(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context,
                                                              aggParsers, suggesters);
                 } catch (ElasticsearchParseException srpe) {
                     throw new ElasticsearchParseException("could not parse [{}] input for watch [{}]. failed to parse [{}]", srpe, TYPE,
@@ -170,7 +170,7 @@ public class SearchInput implements Input {
         return new SearchInput(request, extract, timeout, dynamicNameTimeZone);
     }
 
-    public static Builder builder(SearchRequest request) {
+    public static Builder builder(WatcherSearchTemplateRequest request) {
         return new Builder(request);
     }
 
@@ -198,20 +198,19 @@ public class SearchInput implements Input {
                 return builder;
             }
             builder.startObject(type);
-            builder.field(Field.REQUEST.getPreferredName());
+            builder.field(Field.REQUEST.getPreferredName(), new WatcherSearchTemplateRequest(request));
-            WatcherUtils.writeSearchRequest(request, builder, params);
             return builder.endObject();
         }
     }
 
     public static class Builder implements Input.Builder<SearchInput> {
 
-        private final SearchRequest request;
+        private final WatcherSearchTemplateRequest request;
         private final Set<String> extractKeys = new HashSet<>();
         private TimeValue timeout;
         private DateTimeZone dynamicNameTimeZone;
 
-        private Builder(SearchRequest request) {
+        private Builder(WatcherSearchTemplateRequest request) {
             this.request = request;
         }
 

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java

@@ -5,8 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.input.search;
 
-import java.io.IOException;
-
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.Loggers;
@@ -17,10 +15,14 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.xpack.common.ScriptServiceProxy;
 import org.elasticsearch.xpack.security.InternalClient;
 import org.elasticsearch.xpack.watcher.input.InputFactory;
 import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
+
+import java.io.IOException;
 
 /**
  *
@@ -33,15 +35,16 @@ public class SearchInputFactory extends InputFactory<SearchInput, SearchInput.Re
     private final AggregatorParsers aggParsers;
     private final Suggesters suggesters;
     private final ParseFieldMatcher parseFieldMatcher;
+    private final WatcherSearchTemplateService searchTemplateService;
 
     @Inject
     public SearchInputFactory(Settings settings, InternalClient client, IndicesQueriesRegistry queryRegistry,
-                              AggregatorParsers aggParsers, Suggesters suggesters) {
+                              AggregatorParsers aggParsers, Suggesters suggesters, ScriptServiceProxy scriptService) {
-        this(settings, new WatcherClientProxy(settings, client), queryRegistry, aggParsers, suggesters);
+        this(settings, new WatcherClientProxy(settings, client), queryRegistry, aggParsers, suggesters, scriptService);
     }
 
     public SearchInputFactory(Settings settings, WatcherClientProxy client, IndicesQueriesRegistry queryRegistry,
-                              AggregatorParsers aggParsers, Suggesters suggesters) {
+                              AggregatorParsers aggParsers, Suggesters suggesters, ScriptServiceProxy scriptService) {
         super(Loggers.getLogger(ExecutableSimpleInput.class, settings));
         this.parseFieldMatcher = new ParseFieldMatcher(settings);
         this.client = client;
@@ -49,6 +52,7 @@ public class SearchInputFactory extends InputFactory<SearchInput, SearchInput.Re
         this.aggParsers = aggParsers;
         this.suggesters = suggesters;
         this.defaultTimeout = settings.getAsTime("xpack.watcher.input.search.default_timeout", null);
+        this.searchTemplateService = new WatcherSearchTemplateService(settings, scriptService, queryRegistry, aggParsers, suggesters);
     }
 
     @Override
@@ -64,6 +68,6 @@ public class SearchInputFactory extends InputFactory<SearchInput, SearchInput.Re
 
     @Override
     public ExecutableSearchInput createExecutable(SearchInput input) {
-        return new ExecutableSearchInput(input, inputLogger, client, defaultTimeout);
+        return new ExecutableSearchInput(input, inputLogger, client, searchTemplateService, defaultTimeout);
     }
 }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java

@@ -33,6 +33,9 @@ public class RestAckWatchAction extends WatcherRestHandler {
         super(settings, client);
         controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/watch/{id}/_ack", this);
         controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/watch/{id}/_ack", this);
+        controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/watch/{id}/_ack/{actions}", this);
+        controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/watch/{id}/_ack/{actions}", this);
+        // these are going to be removed in 6.0
         controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/watch/{id}/{actions}/_ack", this);
         controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/watch/{id}/{actions}/_ack", this);
     }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.support;
 
-import com.google.common.base.Charsets;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -21,9 +20,10 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.security.InternalClient;
-import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
 import org.elasticsearch.xpack.template.TemplateUtils;
+import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
 
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.Executor;
@@ -168,7 +168,7 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
         }
         executor.execute(() -> {
             final byte[] template = TemplateUtils.loadTemplate("/" + config.getFileName()+ ".json", INDEX_TEMPLATE_VERSION,
-                    Pattern.quote("${xpack.watcher.template.version}")).getBytes(Charsets.UTF_8);
+                    Pattern.quote("${xpack.watcher.template.version}")).getBytes(StandardCharsets.UTF_8);
 
             PutIndexTemplateRequest request = new PutIndexTemplateRequest(config.getTemplateName()).source(template);
             request.masterNodeTimeout(TimeValue.timeValueMinutes(1));

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherUtils.java

@@ -5,54 +5,22 @@
  */
 package org.elasticsearch.xpack.watcher.support;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.formatDate;
-
-import java.io.IOException;
-import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.Template;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.suggest.Suggesters;
-import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
-import org.elasticsearch.xpack.watcher.watch.Payload;
 import org.joda.time.DateTime;
 
-/**
+import java.io.IOException;
- */
+import java.lang.reflect.Array;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.formatDate;
+
 public final class WatcherUtils {
 
-    static final ParseField INDICES_FIELD = new ParseField("indices");
-    static final ParseField TYPES_FIELD = new ParseField("types");
-    static final ParseField BODY_FIELD = new ParseField("body");
-    static final ParseField SEARCH_TYPE_FIELD = new ParseField("search_type");
-    static final ParseField INDICES_OPTIONS_FIELD = new ParseField("indices_options");
-    static final ParseField EXPAND_WILDCARDS_FIELD = new ParseField("expand_wildcards");
-    static final ParseField IGNORE_UNAVAILABLE_FIELD = new ParseField("ignore_unavailable");
-    static final ParseField ALLOW_NO_INDICES_FIELD = new ParseField("allow_no_indices");
-    static final ParseField TEMPLATE_FIELD = new ParseField("template");
-
-    public final static IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.lenientExpandOpen();
 
     private WatcherUtils() {
     }
@@ -62,214 +30,6 @@ public final class WatcherUtils {
         return XContentHelper.convertToMap(builder.bytes(), false).v2();
     }
 
-    public static SearchRequest createSearchRequestFromPrototype(SearchRequest requestPrototype, WatchExecutionContext ctx,
-                                                                 Payload payload) throws IOException {
-        SearchRequest request = new SearchRequest()
-                .indicesOptions(requestPrototype.indicesOptions())
-                .searchType(requestPrototype.searchType())
-                .indices(requestPrototype.indices())
-                .types(requestPrototype.types());
-
-        // Due the inconsistency with templates in ES 1.x, we maintain our own template format.
-        // This template format we use now, will become the template structure in ES 2.0
-        Map<String, Object> watcherContextParams = Variables.createCtxModel(ctx, payload);
-        if (requestPrototype.source() != null) {
-            // Here we convert a watch search request body into an inline search template,
-            // this way if any Watcher related context variables are used, they will get resolved,
-            // by ES search template support
-            XContentBuilder builder = jsonBuilder();
-            requestPrototype.source().toXContent(builder, ToXContent.EMPTY_PARAMS);
-            Template template = new Template(builder.string(), ScriptType.INLINE, null, builder.contentType(), watcherContextParams);
-            request.template(template);
-        } else if (requestPrototype.template() != null) {
-            // Here we convert watcher template into a ES core templates. Due to the different format we use, we
-            // convert to the template format used in ES core
-            Template template = requestPrototype.template();
-            if (template.getParams() != null) {
-                watcherContextParams.putAll(template.getParams());
-            }
-            template = new Template(template.getScript(), template.getType(), template.getLang(), template.getContentType(),
-                    watcherContextParams);
-            request.template(template);
-            // }
-        }
-        // falling back to an empty body
-        return request;
-    }
-
-
-    /**
-     * Reads a new search request instance for the specified parser.
-     */
-    public static SearchRequest readSearchRequest(XContentParser parser, SearchType searchType, QueryParseContext context,
-                                                  AggregatorParsers aggParsers, Suggesters suggesters)
-            throws IOException {
-        IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
-        SearchRequest searchRequest = new SearchRequest();
-
-        XContentParser.Token token;
-        String currentFieldName = null;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-                if (ParseFieldMatcher.STRICT.match(currentFieldName, BODY_FIELD)) {
-                    searchRequest.source(SearchSourceBuilder.fromXContent(context, aggParsers, suggesters));
-                }
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_FIELD)) {
-                    List<String> indices = new ArrayList<>();
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        if (token == XContentParser.Token.VALUE_STRING) {
-                            indices.add(parser.textOrNull());
-                        } else {
-                            throw new ElasticsearchParseException("could not read search request. expected string values in [" +
-                                    currentFieldName + "] field, but instead found [" + token + "]");
-                        }
-                    }
-                    searchRequest.indices(indices.toArray(new String[indices.size()]));
-                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TYPES_FIELD)) {
-                    List<String> types = new ArrayList<>();
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        if (token == XContentParser.Token.VALUE_STRING) {
-                            types.add(parser.textOrNull());
-                        } else {
-                            throw new ElasticsearchParseException("could not read search request. expected string values in [" +
-                                    currentFieldName + "] field, but instead found [" + token + "]");
-                        }
-                    }
-                    searchRequest.types(types.toArray(new String[types.size()]));
-                } else {
-                    throw new ElasticsearchParseException("could not read search request. unexpected array field [" +
-                            currentFieldName + "]");
-                }
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_OPTIONS_FIELD)) {
-                    boolean expandOpen = DEFAULT_INDICES_OPTIONS.expandWildcardsOpen();
-                    boolean expandClosed = DEFAULT_INDICES_OPTIONS.expandWildcardsClosed();
-                    boolean allowNoIndices = DEFAULT_INDICES_OPTIONS.allowNoIndices();
-                    boolean ignoreUnavailable = DEFAULT_INDICES_OPTIONS.ignoreUnavailable();
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            currentFieldName = parser.currentName();
-                        } else if (token.isValue()) {
-                            if (ParseFieldMatcher.STRICT.match(currentFieldName, EXPAND_WILDCARDS_FIELD)) {
-                                switch (parser.text()) {
-                                    case "all":
-                                        expandOpen = true;
-                                        expandClosed = true;
-                                        break;
-                                    case "open":
-                                        expandOpen = true;
-                                        expandClosed = false;
-                                        break;
-                                    case "closed":
-                                        expandOpen = false;
-                                        expandClosed = true;
-                                        break;
-                                    case "none":
-                                        expandOpen = false;
-                                        expandClosed = false;
-                                        break;
-                                    default:
-                                        throw new ElasticsearchParseException("could not read search request. unknown value [" +
-                                                parser.text() + "] for [" + currentFieldName + "] field ");
-                                }
-                            } else if (ParseFieldMatcher.STRICT.match(currentFieldName, IGNORE_UNAVAILABLE_FIELD)) {
-                                ignoreUnavailable = parser.booleanValue();
-                            } else if (ParseFieldMatcher.STRICT.match(currentFieldName, ALLOW_NO_INDICES_FIELD)) {
-                                allowNoIndices = parser.booleanValue();
-                            } else {
-                                throw new ElasticsearchParseException("could not read search request. unexpected index option [" +
-                                        currentFieldName + "]");
-                            }
-                        } else {
-                            throw new ElasticsearchParseException("could not read search request. unexpected object field [" +
-                                    currentFieldName + "]");
-                        }
-                    }
-                    indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandOpen, expandClosed,
-                            DEFAULT_INDICES_OPTIONS);
-                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TEMPLATE_FIELD)) {
-                    Template template = Template.parse(parser, ParseFieldMatcher.STRICT);
-                    searchRequest.template(template);
-                } else {
-                    throw new ElasticsearchParseException("could not read search request. unexpected object field [" +
-                            currentFieldName + "]");
-                }
-            } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_FIELD)) {
-                    String indicesStr = parser.text();
-                    searchRequest.indices(Strings.delimitedListToStringArray(indicesStr, ",", " \t"));
-                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TYPES_FIELD)) {
-                    String typesStr = parser.text();
-                    searchRequest.types(Strings.delimitedListToStringArray(typesStr, ",", " \t"));
-                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, SEARCH_TYPE_FIELD)) {
-                    searchType = SearchType.fromString(parser.text().toLowerCase(Locale.ROOT), ParseFieldMatcher.EMPTY);
-                } else {
-                    throw new ElasticsearchParseException("could not read search request. unexpected string field [" +
-                            currentFieldName + "]");
-                }
-            } else {
-                throw new ElasticsearchParseException("could not read search request. unexpected token [" + token + "]");
-            }
-        }
-
-        if (searchRequest.indices() == null) {
-            searchRequest.indices(Strings.EMPTY_ARRAY);
-        }
-        searchRequest.searchType(searchType);
-        searchRequest.indicesOptions(indicesOptions);
-        return searchRequest;
-    }
-
-    /**
-     * Writes the searchRequest to the specified builder.
-     */
-    public static XContentBuilder writeSearchRequest(SearchRequest searchRequest, XContentBuilder builder,
-                                                     ToXContent.Params params) throws IOException {
-        if (searchRequest == null) {
-            builder.nullValue();
-            return builder;
-        }
-
-        builder.startObject();
-        if (searchRequest.searchType() != null) {
-            builder.field(SEARCH_TYPE_FIELD.getPreferredName(), searchRequest.searchType().toString().toLowerCase(Locale.ENGLISH));
-        }
-        if (searchRequest.indices() != null) {
-            builder.array(INDICES_FIELD.getPreferredName(), searchRequest.indices());
-        }
-        if (searchRequest.types() != null) {
-            builder.array(TYPES_FIELD.getPreferredName(), searchRequest.types());
-        }
-        if (searchRequest.source() != null) {
-            builder.field(BODY_FIELD.getPreferredName(), searchRequest.source());
-        }
-        if (searchRequest.template() != null) {
-            builder.field(TEMPLATE_FIELD.getPreferredName(), searchRequest.template());
-        }
-
-        if (searchRequest.indicesOptions() != DEFAULT_INDICES_OPTIONS) {
-            IndicesOptions options = searchRequest.indicesOptions();
-            builder.startObject(INDICES_OPTIONS_FIELD.getPreferredName());
-            String value;
-            if (options.expandWildcardsClosed() && options.expandWildcardsOpen()) {
-                value = "all";
-            } else if (options.expandWildcardsOpen()) {
-                value = "open";
-            } else if (options.expandWildcardsClosed()) {
-                value = "closed";
-            } else {
-                value = "none";
-            }
-            builder.field(EXPAND_WILDCARDS_FIELD.getPreferredName(), value);
-            builder.field(IGNORE_UNAVAILABLE_FIELD.getPreferredName(), options.ignoreUnavailable());
-            builder.field(ALLOW_NO_INDICES_FIELD.getPreferredName(), options.allowNoIndices());
-            builder.endObject();
-        }
-        return builder.endObject();
-    }
-
     public static Map<String, Object> flattenModel(Map<String, Object> map) {
         Map<String, Object> result = new HashMap<>();
         flattenModel("", map, result);

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java

@@ -0,0 +1,254 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher.support.search;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.xpack.watcher.support.Script;
+import org.elasticsearch.xpack.watcher.support.SearchRequestEquivalence;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * A {@link WatcherSearchTemplateRequest} contains the search request and the eventual template that will
+ * be rendered as a script by {@link WatcherSearchTemplateService} before being executed.
+ */
+public class WatcherSearchTemplateRequest implements ToXContent {
+
+    private final SearchRequest request;
+    private final @Nullable Script template;
+
+    public WatcherSearchTemplateRequest(SearchRequest searchRequest, @Nullable Script template) {
+        this.request = Objects.requireNonNull(searchRequest);
+        this.template = template;
+    }
+
+    public WatcherSearchTemplateRequest(SearchRequest request) {
+        this(request, null);
+    }
+
+    public SearchRequest getRequest() {
+        return request;
+    }
+
+    public Script getTemplate() {
+        return template;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (request != null) {
+            if (request.searchType() != null) {
+                builder.field(SEARCH_TYPE_FIELD.getPreferredName(), request.searchType().toString().toLowerCase(Locale.ENGLISH));
+            }
+            if (request.indices() != null) {
+                builder.array(INDICES_FIELD.getPreferredName(), request.indices());
+            }
+            if (request.types() != null) {
+                builder.array(TYPES_FIELD.getPreferredName(), request.types());
+            }
+            if (request.source() != null) {
+                builder.field(BODY_FIELD.getPreferredName(), request.source());
+            }
+            if (request.indicesOptions() != DEFAULT_INDICES_OPTIONS) {
+                IndicesOptions options = request.indicesOptions();
+                builder.startObject(INDICES_OPTIONS_FIELD.getPreferredName());
+                String value;
+                if (options.expandWildcardsClosed() && options.expandWildcardsOpen()) {
+                    value = "all";
+                } else if (options.expandWildcardsOpen()) {
+                    value = "open";
+                } else if (options.expandWildcardsClosed()) {
+                    value = "closed";
+                } else {
+                    value = "none";
+                }
+                builder.field(EXPAND_WILDCARDS_FIELD.getPreferredName(), value);
+                builder.field(IGNORE_UNAVAILABLE_FIELD.getPreferredName(), options.ignoreUnavailable());
+                builder.field(ALLOW_NO_INDICES_FIELD.getPreferredName(), options.allowNoIndices());
+                builder.endObject();
+            }
+        }
+        if (template != null) {
+            builder.field(TEMPLATE_FIELD.getPreferredName(), template);
+        }
+        return builder.endObject();
+    }
+
+
+    /**
+     * Reads a new watcher search request instance for the specified parser.
+     */
+    public static WatcherSearchTemplateRequest fromXContent(XContentParser parser, SearchType searchType, QueryParseContext context,
+                                                            AggregatorParsers aggParsers, Suggesters suggesters)
+            throws IOException {
+        IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
+        SearchRequest searchRequest = new SearchRequest();
+        Script template = null;
+
+        XContentParser.Token token;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+                if (ParseFieldMatcher.STRICT.match(currentFieldName, BODY_FIELD)) {
+                    searchRequest.source(SearchSourceBuilder.fromXContent(context, aggParsers, suggesters));
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_FIELD)) {
+                    List<String> indices = new ArrayList<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.VALUE_STRING) {
+                            indices.add(parser.textOrNull());
+                        } else {
+                            throw new ElasticsearchParseException("could not read search request. expected string values in [" +
+                                    currentFieldName + "] field, but instead found [" + token + "]");
+                        }
+                    }
+                    searchRequest.indices(indices.toArray(new String[indices.size()]));
+                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TYPES_FIELD)) {
+                    List<String> types = new ArrayList<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.VALUE_STRING) {
+                            types.add(parser.textOrNull());
+                        } else {
+                            throw new ElasticsearchParseException("could not read search request. expected string values in [" +
+                                    currentFieldName + "] field, but instead found [" + token + "]");
+                        }
+                    }
+                    searchRequest.types(types.toArray(new String[types.size()]));
+                } else {
+                    throw new ElasticsearchParseException("could not read search request. unexpected array field [" +
+                            currentFieldName + "]");
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_OPTIONS_FIELD)) {
+                    boolean expandOpen = DEFAULT_INDICES_OPTIONS.expandWildcardsOpen();
+                    boolean expandClosed = DEFAULT_INDICES_OPTIONS.expandWildcardsClosed();
+                    boolean allowNoIndices = DEFAULT_INDICES_OPTIONS.allowNoIndices();
+                    boolean ignoreUnavailable = DEFAULT_INDICES_OPTIONS.ignoreUnavailable();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (token.isValue()) {
+                            if (ParseFieldMatcher.STRICT.match(currentFieldName, EXPAND_WILDCARDS_FIELD)) {
+                                switch (parser.text()) {
+                                    case "all":
+                                        expandOpen = true;
+                                        expandClosed = true;
+                                        break;
+                                    case "open":
+                                        expandOpen = true;
+                                        expandClosed = false;
+                                        break;
+                                    case "closed":
+                                        expandOpen = false;
+                                        expandClosed = true;
+                                        break;
+                                    case "none":
+                                        expandOpen = false;
+                                        expandClosed = false;
+                                        break;
+                                    default:
+                                        throw new ElasticsearchParseException("could not read search request. unknown value [" +
+                                                parser.text() + "] for [" + currentFieldName + "] field ");
+                                }
+                            } else if (ParseFieldMatcher.STRICT.match(currentFieldName, IGNORE_UNAVAILABLE_FIELD)) {
+                                ignoreUnavailable = parser.booleanValue();
+                            } else if (ParseFieldMatcher.STRICT.match(currentFieldName, ALLOW_NO_INDICES_FIELD)) {
+                                allowNoIndices = parser.booleanValue();
+                            } else {
+                                throw new ElasticsearchParseException("could not read search request. unexpected index option [" +
+                                        currentFieldName + "]");
+                            }
+                        } else {
+                            throw new ElasticsearchParseException("could not read search request. unexpected object field [" +
+                                    currentFieldName + "]");
+                        }
+                    }
+                    indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandOpen, expandClosed,
+                            DEFAULT_INDICES_OPTIONS);
+                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TEMPLATE_FIELD)) {
+                    template = Script.parse(parser);
+                } else {
+                    throw new ElasticsearchParseException("could not read search request. unexpected object field [" +
+                            currentFieldName + "]");
+                }
+            } else if (token == XContentParser.Token.VALUE_STRING) {
+                if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_FIELD)) {
+                    String indicesStr = parser.text();
+                    searchRequest.indices(Strings.delimitedListToStringArray(indicesStr, ",", " \t"));
+                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TYPES_FIELD)) {
+                    String typesStr = parser.text();
+                    searchRequest.types(Strings.delimitedListToStringArray(typesStr, ",", " \t"));
+                } else if (ParseFieldMatcher.STRICT.match(currentFieldName, SEARCH_TYPE_FIELD)) {
+                    searchType = SearchType.fromString(parser.text().toLowerCase(Locale.ROOT), ParseFieldMatcher.EMPTY);
+                } else {
+                    throw new ElasticsearchParseException("could not read search request. unexpected string field [" +
+                            currentFieldName + "]");
+                }
+            } else {
+                throw new ElasticsearchParseException("could not read search request. unexpected token [" + token + "]");
+            }
+        }
+
+        if (searchRequest.indices() == null) {
+            searchRequest.indices(Strings.EMPTY_ARRAY);
+        }
+        searchRequest.searchType(searchType);
+        searchRequest.indicesOptions(indicesOptions);
+        return new WatcherSearchTemplateRequest(searchRequest, template);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        WatcherSearchTemplateRequest that = (WatcherSearchTemplateRequest) o;
+
+        if (!SearchRequestEquivalence.INSTANCE.equivalent(request, that.request)) return false;
+        return template != null ? template.equals(that.template) : that.template == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = request != null ? request.hashCode() : 0;
+        result = 31 * result + (template != null ? template.hashCode() : 0);
+        return result;
+    }
+
+    static final ParseField INDICES_FIELD = new ParseField("indices");
+    static final ParseField TYPES_FIELD = new ParseField("types");
+    static final ParseField BODY_FIELD = new ParseField("body");
+    static final ParseField SEARCH_TYPE_FIELD = new ParseField("search_type");
+    static final ParseField INDICES_OPTIONS_FIELD = new ParseField("indices_options");
+    static final ParseField EXPAND_WILDCARDS_FIELD = new ParseField("expand_wildcards");
+    static final ParseField IGNORE_UNAVAILABLE_FIELD = new ParseField("ignore_unavailable");
+    static final ParseField ALLOW_NO_INDICES_FIELD = new ParseField("allow_no_indices");
+    static final ParseField TEMPLATE_FIELD = new ParseField("template");
+
+    public final static IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.lenientExpandOpen();
+}

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher.support.search;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.xpack.common.ScriptServiceProxy;
+import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
+import org.elasticsearch.xpack.watcher.support.Script;
+import org.elasticsearch.xpack.watcher.support.Variables;
+import org.elasticsearch.xpack.watcher.watch.Payload;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ * {@link WatcherSearchTemplateService} renders {@link WatcherSearchTemplateRequest} before their execution.
+ */
+public class WatcherSearchTemplateService extends AbstractComponent {
+
+    private static final String DEFAULT_LANG = "mustache";
+
+    private final ScriptServiceProxy scriptService;
+    private final ParseFieldMatcher parseFieldMatcher;
+    private final IndicesQueriesRegistry queryRegistry;
+    private final AggregatorParsers aggsParsers;
+    private final Suggesters suggesters;
+
+    @Inject
+    public WatcherSearchTemplateService(Settings settings, ScriptServiceProxy scriptServiceProxy,
+                                        IndicesQueriesRegistry queryRegistry, AggregatorParsers aggregatorParsers, Suggesters suggesters) {
+        super(settings);
+        this.scriptService = scriptServiceProxy;
+        this.queryRegistry = queryRegistry;
+        this.aggsParsers = aggregatorParsers;
+        this.suggesters = suggesters;
+        this.parseFieldMatcher = new ParseFieldMatcher(settings);
+    }
+
+    public SearchRequest createSearchRequestFromPrototype(WatcherSearchTemplateRequest prototype, WatchExecutionContext ctx,
+                                                          Payload payload) throws IOException {
+
+        SearchRequest request = new SearchRequest()
+                .indicesOptions(prototype.getRequest().indicesOptions())
+                .searchType(prototype.getRequest().searchType())
+                .indices(prototype.getRequest().indices())
+                .types(prototype.getRequest().types());
+
+        Script template = null;
+
+        // Due the inconsistency with templates in ES 1.x, we maintain our own template format.
+        // This template format we use now, will become the template structure in ES 2.0
+        Map<String, Object> watcherContextParams = Variables.createCtxModel(ctx, payload);
+
+        // Here we convert a watch search request body into an inline search template,
+        // this way if any Watcher related context variables are used, they will get resolved.
+        if (prototype.getRequest().source() != null) {
+            try (XContentBuilder builder = jsonBuilder()) {
+                prototype.getRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS);
+                template = Script.inline(builder.string()).lang(DEFAULT_LANG).params(watcherContextParams).build();
+            }
+
+        } else if (prototype.getTemplate() != null) {
+            // Here we convert watcher template into a ES core templates. Due to the different format we use, we
+            // convert to the template format used in ES core
+            Script templatePrototype = prototype.getTemplate();
+            if (templatePrototype.params() != null) {
+                watcherContextParams.putAll(templatePrototype.params());
+            }
+
+            Script.Builder builder;
+            if (templatePrototype.type() == ScriptService.ScriptType.INLINE) {
+                builder = Script.inline(templatePrototype.script());
+            } else if (templatePrototype.type() == ScriptService.ScriptType.FILE) {
+                builder = Script.file(templatePrototype.script());
+            } else if (templatePrototype.type() == ScriptService.ScriptType.STORED) {
+                builder = Script.indexed(templatePrototype.script());
+            } else {
+                builder = Script.defaultType(templatePrototype.script());
+            }
+            template = builder.lang(templatePrototype.lang()).params(watcherContextParams).build();
+        }
+
+        request.source(convert(template));
+        return request;
+    }
+
+    /**
+     * Converts a {@link Script} to a {@link org.elasticsearch.search.builder.SearchSourceBuilder}
+     */
+    private SearchSourceBuilder convert(Script template) throws IOException {
+        SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
+        if (template == null) {
+            // falling back to an empty body
+            return sourceBuilder;
+        }
+
+        BytesReference source = (BytesReference) scriptService.executable(scriptService.compile(template), template.params()).run();
+        if (source != null && source.length() > 0) {
+            try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
+                sourceBuilder.parseXContent(new QueryParseContext(queryRegistry, parser, parseFieldMatcher), aggsParsers, suggesters);
+            }
+        }
+        return sourceBuilder;
+    }
+}

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/TransformBuilders.java

@@ -6,8 +6,8 @@
 package org.elasticsearch.xpack.watcher.transform;
 
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.xpack.watcher.support.Script;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.transform.chain.ChainTransform;
 import org.elasticsearch.xpack.watcher.transform.script.ScriptTransform;
 import org.elasticsearch.xpack.watcher.transform.search.SearchTransform;
@@ -20,12 +20,12 @@ public final class TransformBuilders {
     private TransformBuilders() {
     }
 
-    public static SearchTransform.Builder searchTransform(SearchRequest request) {
+    public static SearchTransform.Builder searchTransform(WatcherSearchTemplateRequest request) {
         return SearchTransform.builder(request);
     }
 
-    public static SearchTransform.Builder searchTransform(SearchRequestBuilder request) {
+    public static SearchTransform.Builder searchTransform(SearchRequest request) {
-        return searchTransform(request.request());
+        return searchTransform(new WatcherSearchTemplateRequest(request));
     }
 
     public static ScriptTransform.Builder scriptTransform(String script) {

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/ExecutableSearchTransform.java

@@ -12,8 +12,8 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
-import org.elasticsearch.xpack.watcher.support.WatcherUtils;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
 import org.elasticsearch.xpack.watcher.transform.ExecutableTransform;
 import org.elasticsearch.xpack.watcher.watch.Payload;
 
@@ -25,12 +25,14 @@ public class ExecutableSearchTransform extends ExecutableTransform<SearchTransfo
     public static final SearchType DEFAULT_SEARCH_TYPE = SearchType.QUERY_THEN_FETCH;
 
     protected final WatcherClientProxy client;
+    private final WatcherSearchTemplateService searchTemplateService;
     protected final @Nullable TimeValue timeout;
 
     public ExecutableSearchTransform(SearchTransform transform, ESLogger logger, WatcherClientProxy client,
-                                     @Nullable TimeValue defaultTimeout) {
+                                     WatcherSearchTemplateService searchTemplateService, @Nullable TimeValue defaultTimeout) {
         super(transform, logger);
         this.client = client;
+        this.searchTemplateService = searchTemplateService;
         this.timeout = transform.getTimeout() != null ? transform.getTimeout() : defaultTimeout;
     }
 
@@ -38,7 +40,7 @@ public class ExecutableSearchTransform extends ExecutableTransform<SearchTransfo
     public SearchTransform.Result execute(WatchExecutionContext ctx, Payload payload) {
         SearchRequest request = null;
         try {
-            request = WatcherUtils.createSearchRequestFromPrototype(transform.getRequest(), ctx, payload);
+            request = searchTemplateService.createSearchRequestFromPrototype(transform.getRequest(), ctx, payload);
             SearchResponse resp = client.search(request, timeout);
             return new SearchTransform.Result(request, new Payload.XContent(resp));
         } catch (Exception e) {

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransform.java

@@ -16,9 +16,8 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.suggest.Suggesters;
-import org.elasticsearch.xpack.watcher.support.SearchRequestEquivalence;
 import org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils;
-import org.elasticsearch.xpack.watcher.support.WatcherUtils;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.transform.Transform;
 import org.elasticsearch.xpack.watcher.watch.Payload;
 import org.joda.time.DateTimeZone;
@@ -32,11 +31,11 @@ public class SearchTransform implements Transform {
 
     public static final String TYPE = "search";
 
-    private final SearchRequest request;
+    private final WatcherSearchTemplateRequest request;
     private final @Nullable TimeValue timeout;
     private final @Nullable DateTimeZone dynamicNameTimeZone;
 
-    public SearchTransform(SearchRequest request, @Nullable TimeValue timeout, @Nullable DateTimeZone dynamicNameTimeZone) {
+    public SearchTransform(WatcherSearchTemplateRequest request, @Nullable TimeValue timeout, @Nullable DateTimeZone dynamicNameTimeZone) {
         this.request = request;
         this.timeout = timeout;
         this.dynamicNameTimeZone = dynamicNameTimeZone;
@@ -47,7 +46,7 @@ public class SearchTransform implements Transform {
         return TYPE;
     }
 
-    public SearchRequest getRequest() {
+    public WatcherSearchTemplateRequest getRequest() {
         return request;
     }
 
@@ -66,14 +65,14 @@ public class SearchTransform implements Transform {
 
         SearchTransform that = (SearchTransform) o;
 
-        if (!SearchRequestEquivalence.INSTANCE.equivalent(request, this.request)) return false;
+        if (request != null ? !request.equals(that.request) : that.request != null) return false;
         if (timeout != null ? !timeout.equals(that.timeout) : that.timeout != null) return false;
         return !(dynamicNameTimeZone != null ? !dynamicNameTimeZone.equals(that.dynamicNameTimeZone) : that.dynamicNameTimeZone != null);
     }
 
     @Override
     public int hashCode() {
-        int result = request.hashCode();
+        int result = request != null ? request.hashCode() : 0;
         result = 31 * result + (timeout != null ? timeout.hashCode() : 0);
         result = 31 * result + (dynamicNameTimeZone != null ? dynamicNameTimeZone.hashCode() : 0);
         return result;
@@ -82,8 +81,9 @@ public class SearchTransform implements Transform {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(Field.REQUEST.getPreferredName());
+        if (request != null) {
-        builder = WatcherUtils.writeSearchRequest(request, builder, params);
+            builder.field(Field.REQUEST.getPreferredName(), request);
+        }
         if (timeout != null) {
             builder.field(Field.TIMEOUT.getPreferredName(), timeout);
         }
@@ -97,7 +97,7 @@ public class SearchTransform implements Transform {
     public static SearchTransform parse(String watchId, XContentParser parser, QueryParseContext context,
                                         AggregatorParsers aggParsers, Suggesters suggesters)
             throws IOException {
-        SearchRequest request = null;
+        WatcherSearchTemplateRequest request = null;
         TimeValue timeout = null;
         DateTimeZone dynamicNameTimeZone = null;
 
@@ -108,7 +108,7 @@ public class SearchTransform implements Transform {
                 currentFieldName = parser.currentName();
             } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Field.REQUEST)) {
                 try {
-                    request = WatcherUtils.readSearchRequest(parser, ExecutableSearchTransform.DEFAULT_SEARCH_TYPE, context,
+                    request = WatcherSearchTemplateRequest.fromXContent(parser, ExecutableSearchTransform.DEFAULT_SEARCH_TYPE, context,
                                                              aggParsers, suggesters);
                 } catch (ElasticsearchParseException srpe) {
                     throw new ElasticsearchParseException("could not parse [{}] transform for watch [{}]. failed to parse [{}]", srpe,
@@ -136,7 +136,7 @@ public class SearchTransform implements Transform {
         return new SearchTransform(request, timeout, dynamicNameTimeZone);
     }
 
-    public static Builder builder(SearchRequest request) {
+    public static Builder builder(WatcherSearchTemplateRequest request) {
         return new Builder(request);
     }
 
@@ -162,8 +162,7 @@ public class SearchTransform implements Transform {
         protected XContentBuilder typeXContent(XContentBuilder builder, Params params) throws IOException {
             if (request != null) {
                 builder.startObject(type);
-                builder.field(Field.REQUEST.getPreferredName());
+                builder.field(Field.REQUEST.getPreferredName(), new WatcherSearchTemplateRequest(request));
-                WatcherUtils.writeSearchRequest(request, builder, params);
                 builder.endObject();
             }
             return builder;
@@ -172,11 +171,11 @@ public class SearchTransform implements Transform {
 
     public static class Builder implements Transform.Builder<SearchTransform> {
 
-        private final SearchRequest request;
+        private final WatcherSearchTemplateRequest request;
         private TimeValue timeout;
         private DateTimeZone dynamicNameTimeZone;
 
-        public Builder(SearchRequest request) {
+        public Builder(WatcherSearchTemplateRequest request) {
             this.request = request;
         }
 

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java

@@ -18,7 +18,9 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.xpack.security.InternalClient;
+import org.elasticsearch.xpack.common.ScriptServiceProxy;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
 import org.elasticsearch.xpack.watcher.transform.TransformFactory;
 
 /**
@@ -32,14 +34,15 @@ public class SearchTransformFactory extends TransformFactory<SearchTransform, Se
     private final AggregatorParsers aggParsers;
     private final Suggesters suggesters;
     private final ParseFieldMatcher parseFieldMatcher;
+    private final WatcherSearchTemplateService searchTemplateService;
 
     @Inject
     public SearchTransformFactory(Settings settings, InternalClient client, IndicesQueriesRegistry queryRegistry,
-                                  AggregatorParsers aggParsers, Suggesters suggesters) {
+                                  AggregatorParsers aggParsers, Suggesters suggesters, ScriptServiceProxy scriptService) {
-        this(settings, new WatcherClientProxy(settings, client), queryRegistry, aggParsers, suggesters);
+        this(settings, new WatcherClientProxy(settings, client), queryRegistry, aggParsers, suggesters, scriptService);
     }
     public SearchTransformFactory(Settings settings, WatcherClientProxy client, IndicesQueriesRegistry queryRegistry,
-                                  AggregatorParsers aggParsers, Suggesters suggesters) {
+                                  AggregatorParsers aggParsers, Suggesters suggesters, ScriptServiceProxy scriptService) {
         super(Loggers.getLogger(ExecutableSearchTransform.class, settings));
         this.client = client;
         this.parseFieldMatcher = new ParseFieldMatcher(settings);
@@ -47,6 +50,7 @@ public class SearchTransformFactory extends TransformFactory<SearchTransform, Se
         this.aggParsers = aggParsers;
         this.suggesters = suggesters;
         this.defaultTimeout = settings.getAsTime("xpack.watcher.transform.search.default_timeout", null);
+        this.searchTemplateService = new WatcherSearchTemplateService(settings, scriptService, queryRegistry, aggParsers, suggesters);
     }
 
     @Override
@@ -62,6 +66,6 @@ public class SearchTransformFactory extends TransformFactory<SearchTransform, Se
 
     @Override
     public ExecutableSearchTransform createExecutable(SearchTransform transform) {
-        return new ExecutableSearchTransform(transform, transformLogger, client, defaultTimeout);
+        return new ExecutableSearchTransform(transform, transformLogger, client,  searchTemplateService, defaultTimeout);
     }
 }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/integration/HistoryIntegrationTests.java

@@ -38,7 +38,7 @@ public class HistoryIntegrationTests extends AbstractWatcherIntegrationTestCase
 
         SearchRequestBuilder searchRequestBuilder = client().prepareSearch("foo").addSort(SortBuilders.fieldSort("inner.date").order
                 (SortOrder.DESC));
-        builder.input(InputBuilders.chainInput().add("first", InputBuilders.searchInput(searchRequestBuilder)));
+        builder.input(InputBuilders.chainInput().add("first", InputBuilders.searchInput(searchRequestBuilder.request())));
 
         PutWatchResponse response = watcherClient().preparePutWatch("test_watch").setSource(builder).get();
         assertThat(response.isCreated(), is(true));

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginDisableTests.java

@@ -6,10 +6,9 @@
 package org.elasticsearch.xpack.watcher;
 
 import org.apache.http.HttpStatus;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.http.HttpServerTransport;
@@ -18,12 +17,10 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolInfo;
-import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor;
 import org.elasticsearch.xpack.XPackPlugin;
+import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -70,12 +67,11 @@ public class WatcherPluginDisableTests extends ESIntegTestCase {
 
     public void testRestEndpoints() throws Exception {
         HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class);
-        try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+        try {
-            HttpRequestBuilder request = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport)
+            getRestClient().performRequest("GET", "/_xpack/watcher", Collections.emptyMap(), null);
-                    .method("GET")
+            fail("request should have failed");
-                    .path("/_xpack/watcher");
+        } catch(ResponseException e) {
-            HttpResponse response = request.execute();
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(HttpStatus.SC_BAD_REQUEST));
-            assertThat(response.getStatusCode(), is(HttpStatus.SC_BAD_REQUEST));
         }
     }
 

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.actions.email;
 
-import com.google.common.base.Charsets;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.MapBuilder;
@@ -55,6 +54,7 @@ import org.junit.Before;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -478,7 +478,7 @@ public class EmailActionTests extends ESTestCase {
         Attachment externalAttachment = attachments.get(attachmentId);
         assertThat(externalAttachment.bodyPart(), is(notNullValue()));
         InputStream is = externalAttachment.bodyPart().getInputStream();
-        String data = Streams.copyToString(new InputStreamReader(is, Charsets.UTF_8));
+        String data = Streams.copyToString(new InputStreamReader(is, StandardCharsets.UTF_8));
         assertThat(data, is(content));
     }
 
@@ -610,7 +610,8 @@ public class EmailActionTests extends ESTestCase {
             when(httpClient.execute(any(HttpRequest.class))).thenReturn(mockResponse);
 
             HttpRequestTemplate template = HttpRequestTemplate.builder("localhost", 1234).build();
-            attachments.add(new HttpRequestAttachment(randomAsciiOfLength(10), template, randomFrom("my/custom-type", null)));
+            attachments.add(new HttpRequestAttachment(randomAsciiOfLength(10), template,
+                    randomBoolean(), randomFrom("my/custom-type", null)));
         } else if ("data".equals(attachmentType)) {
             attachments.add(new org.elasticsearch.xpack.notification.email.attachment.DataAttachment(randomAsciiOfLength(10),
                     randomFrom(DataAttachment.JSON, DataAttachment.YAML)));

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailAttachmentTests.java

@@ -207,7 +207,4 @@ public class EmailAttachmentTests extends AbstractWatcherIntegrationTestCase {
             fail("waited too long for email to be received");
         }
     }
-
-
-
 }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.watcher.actions.index;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -64,14 +65,8 @@ public class IndexActionTests extends ESIntegTestCase {
     }
 
     public void testIndexActionExecuteSingleDoc() throws Exception {
-        String timestampField = randomFrom(null, "_timestamp", "@timestamp");
+        String timestampField = randomFrom(null, "@timestamp");
-        boolean customTimestampField = "@timestamp".equals(timestampField);
+        boolean customTimestampField = timestampField != null;
-
-        if (timestampField == null || "_timestamp".equals(timestampField)) {
-            assertThat(prepareCreate("test-index")
-                    .addMapping("test-type", "{ \"test-type\" : { \"_timestamp\" : { \"enabled\" : \"true\" }}}")
-                    .get().isAcknowledged(), is(true));
-        }
 
         IndexAction action = new IndexAction("test-index", "test-type", timestampField, null, null);
         ExecutableIndexAction executable = new ExecutableIndexAction(action, logger, WatcherClientProxy.of(client()), null);
@@ -92,12 +87,15 @@ public class IndexActionTests extends ESIntegTestCase {
 
         refresh(); //Manually refresh to make sure data is available
 
-        SearchResponse searchResponse = client().prepareSearch("test-index")
+        SearchRequestBuilder searchRequestbuilder = client().prepareSearch("test-index")
                 .setTypes("test-type")
-                .setSource(searchSource()
+                .setSource(searchSource().query(matchAllQuery()));
-                        .query(matchAllQuery())
+
-                        .aggregation(terms("timestamps").field(customTimestampField ? timestampField : "_timestamp")))
+        if (customTimestampField) {
-                .get();
+            searchRequestbuilder.addAggregation(terms("timestamps").field(timestampField));
+        }
+
+        SearchResponse searchResponse = searchRequestbuilder.get();
 
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         SearchHit hit = searchResponse.getHits().getAt(0);
@@ -106,28 +104,24 @@ public class IndexActionTests extends ESIntegTestCase {
             assertThat(hit.getSource().size(), is(2));
             assertThat(hit.getSource(), hasEntry("foo", (Object) "bar"));
             assertThat(hit.getSource(), hasEntry(timestampField, (Object) WatcherDateTimeUtils.formatDate(executionTime)));
+
+            Terms terms = searchResponse.getAggregations().get("timestamps");
+            assertThat(terms, notNullValue());
+            assertThat(terms.getBuckets(), hasSize(1));
+            assertThat(terms.getBuckets().get(0).getKeyAsNumber().longValue(), is(executionTime.getMillis()));
+            assertThat(terms.getBuckets().get(0).getDocCount(), is(1L));
         } else {
             assertThat(hit.getSource().size(), is(1));
             assertThat(hit.getSource(), hasEntry("foo", (Object) "bar"));
         }
-        Terms terms = searchResponse.getAggregations().get("timestamps");
-        assertThat(terms, notNullValue());
-        assertThat(terms.getBuckets(), hasSize(1));
-        assertThat(terms.getBuckets().get(0).getKeyAsNumber().longValue(), is(executionTime.getMillis()));
-        assertThat(terms.getBuckets().get(0).getDocCount(), is(1L));
     }
 
     public void testIndexActionExecuteMultiDoc() throws Exception {
-        String timestampField = randomFrom(null, "_timestamp", "@timestamp");
+        String timestampField = randomFrom(null, "@timestamp");
         boolean customTimestampField = "@timestamp".equals(timestampField);
 
-        if (timestampField == null || "_timestamp".equals(timestampField)) {
+        assertAcked(prepareCreate("test-index")
-            assertAcked(prepareCreate("test-index")
+                .addMapping("test-type", "foo", "type=keyword"));
-                    .addMapping("test-type", "_timestamp", "enabled=true", "foo", "type=keyword"));
-        } else {
-            assertAcked(prepareCreate("test-index")
-                    .addMapping("test-type", "foo", "type=keyword"));
-        }
 
         Object list = randomFrom(
                 new Map[] { singletonMap("foo", "bar"), singletonMap("foo", "bar1") },
@@ -160,8 +154,7 @@ public class IndexActionTests extends ESIntegTestCase {
         SearchResponse searchResponse = client().prepareSearch("test-index")
                 .setTypes("test-type")
                 .setSource(searchSource().sort("foo", SortOrder.ASC)
-                        .query(matchAllQuery())
+                        .query(matchAllQuery()))
-                        .aggregation(terms("timestamps").field(customTimestampField ? timestampField : "_timestamp")))
                 .get();
 
         assertThat(searchResponse.getHits().totalHits(), equalTo(2L));

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java

@@ -31,6 +31,7 @@ import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
 import org.elasticsearch.xpack.common.secret.SecretService;
 import org.elasticsearch.xpack.common.text.TextTemplate;
 import org.elasticsearch.xpack.common.text.TextTemplateEngine;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine;
 import org.elasticsearch.xpack.watcher.test.WatcherTestUtils;
@@ -238,7 +239,7 @@ public class WebhookActionTests extends ESTestCase {
 
             ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, httpClient, templateEngine);
             String watchId = "test_url_encode" + randomAsciiOfLength(10);
-            Watch watch = createWatch(watchId, mock(WatcherClientProxy.class), "account1");
+            Watch watch = createWatch(watchId, "account1");
             WatchExecutionContext ctx = new TriggeredExecutionContext(watch, new DateTime(UTC),
                     new ScheduleTriggerEvent(watchId, new DateTime(UTC), new DateTime(UTC)), timeValueSeconds(5));
             executable.execute("_id", ctx, new Payload.Simple());
@@ -264,16 +265,16 @@ public class WebhookActionTests extends ESTestCase {
 
         ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, client, templateEngine);
 
-        Watch watch = createWatch(watchId, mock(WatcherClientProxy.class), "account1");
+        Watch watch = createWatch(watchId, "account1");
         WatchExecutionContext ctx = new TriggeredExecutionContext(watch, new DateTime(UTC),
                 new ScheduleTriggerEvent(watchId, new DateTime(UTC), new DateTime(UTC)), timeValueSeconds(5));
         Action.Result result = executable.execute("_id", ctx, new Payload.Simple());
         assertThat(result, Matchers.instanceOf(WebhookAction.Result.Success.class));
     }
 
-    private Watch createWatch(String watchId, WatcherClientProxy client, final String account) throws AddressException, IOException {
+    private Watch createWatch(String watchId, final String account) throws AddressException, IOException {
         return WatcherTestUtils.createTestWatch(watchId,
-                client,
+                mock(WatcherClientProxy.class),
                 ExecuteScenario.Success.client(),
                 new AbstractWatcherIntegrationTestCase.NoopEmailService() {
                     @Override
@@ -286,6 +287,7 @@ public class WebhookActionTests extends ESTestCase {
                         return new EmailSent(account, email);
                     }
                 },
+                mock(WatcherSearchTemplateService.class),
                 logger);
     };
 

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/compare/CompareConditionSearchTests.java

@@ -39,18 +39,14 @@ public class CompareConditionSearchTests extends AbstractWatcherIntegrationTestC
     }
 
     public void testExecuteWithAggs() throws Exception {
-        client().admin().indices().prepareCreate("my-index")
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:00").get();
-                .addMapping("my-type", "_timestamp", "enabled=true")
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:10").get();
-                .get();
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:20").get();
-
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:30").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:00").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:10").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:20").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:30").setSource("{}").get();
         refresh();
 
         SearchResponse response = client().prepareSearch("my-index")
-                .addAggregation(AggregationBuilders.dateHistogram("rate").field("_timestamp")
+                .addAggregation(AggregationBuilders.dateHistogram("rate").field("@timestamp")
                         .dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC))
                 .get();
 
@@ -65,12 +61,12 @@ public class CompareConditionSearchTests extends AbstractWatcherIntegrationTestC
         assertThat(resolvedValues.size(), is(1));
         assertThat(resolvedValues, hasEntry("ctx.payload.aggregations.rate.buckets.0.doc_count", (Object) 4));
 
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:40").setSource("{}").get();
+        client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:40").get();
         refresh();
 
         response = client().prepareSearch("my-index")
                 .addAggregation(AggregationBuilders.dateHistogram("rate")
-                        .field("_timestamp").dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC))
+                        .field("@timestamp").dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC))
                 .get();
 
         ctx = mockExecutionContext("_name", new Payload.XContent(response));

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/condition/script/ScriptConditionSearchTests.java

@@ -1,105 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher.condition.script;
-
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext;
-import org.elasticsearch.xpack.watcher.support.Script;
-import org.elasticsearch.xpack.common.ScriptServiceProxy;
-import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
-import org.elasticsearch.xpack.watcher.test.WatcherTestUtils;
-import org.elasticsearch.xpack.watcher.watch.Payload;
-import org.junit.After;
-import org.junit.Before;
-
-import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.mockExecutionContext;
-import static org.hamcrest.Matchers.is;
-import static org.mockito.Mockito.when;
-
-/**
- */
-@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/724")
-public class ScriptConditionSearchTests extends AbstractWatcherIntegrationTestCase {
-    private ThreadPool tp = null;
-    private ScriptServiceProxy scriptService;
-
-    @Before
-    public void init() throws Exception {
-        tp = new TestThreadPool(ThreadPool.Names.SAME);
-        scriptService = WatcherTestUtils.getScriptServiceProxy(tp);
-    }
-
-    @After
-    public void cleanup() {
-        tp.shutdownNow();
-    }
-
-    public void testExecuteWithAggs() throws Exception {
-        client().admin().indices().prepareCreate("my-index")
-                .addMapping("my-type", "_timestamp", "enabled=true")
-                .get();
-
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:00").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:10").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:20").setSource("{}").get();
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:30").setSource("{}").get();
-        refresh();
-
-        SearchResponse response = client().prepareSearch("my-index")
-                .addAggregation(AggregationBuilders.dateHistogram("rate")
-                        .field("_timestamp").dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC))
-                .get();
-
-        ExecutableScriptCondition condition = new ExecutableScriptCondition(
-                new ScriptCondition(Script.inline("ctx.payload.aggregations.rate.buckets[0]?.doc_count >= 5").build()),
-                logger, scriptService);
-
-        WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response));
-        assertFalse(condition.execute(ctx).met());
-
-        client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:40").setSource("{}").get();
-        refresh();
-
-        response = client().prepareSearch("my-index")
-                .addAggregation(AggregationBuilders.dateHistogram("rate")
-                        .field("_timestamp").dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC))
-                .get();
-
-        ctx = mockExecutionContext("_name", new Payload.XContent(response));
-        assertThat(condition.execute(ctx).met(), is(true));
-    }
-
-    public void testExecuteAccessHits() throws Exception {
-        ExecutableScriptCondition condition = new ExecutableScriptCondition(
-                new ScriptCondition(Script.inline("ctx.payload.hits?.hits[0]?._score == 1.0").build()), logger, scriptService);
-        InternalSearchHit hit = new InternalSearchHit(0, "1", new Text("type"), null);
-        hit.score(1f);
-        hit.shard(new SearchShardTarget("a", new Index("a", "testUUID"), 0));
-
-        InternalSearchResponse internalSearchResponse = new InternalSearchResponse(
-                new InternalSearchHits(new InternalSearchHit[]{hit}, 1L, 1f), null, null, null, false, false);
-        SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 500L, new ShardSearchFailure[0]);
-
-        WatchExecutionContext ctx = mockExecutionContext("_watch_name", new Payload.XContent(response));
-        assertThat(condition.execute(ctx).met(), is(true));
-        hit.score(2f);
-        when(ctx.payload()).thenReturn(new Payload.XContent(response));
-        assertThat(condition.execute(ctx).met(), is(false));
-    }
-}

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchTests.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.watcher.execution;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.test.WatcherTestUtils;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent;
@@ -21,7 +22,8 @@ import static org.hamcrest.Matchers.equalTo;
  */
 public class TriggeredWatchTests extends AbstractWatcherIntegrationTestCase {
     public void testParser() throws Exception {
-        Watch watch = WatcherTestUtils.createTestWatch("fired_test", watcherHttpClient(), noopEmailService(), logger);
+        Watch watch = WatcherTestUtils.createTestWatch("fired_test", watcherHttpClient(), noopEmailService(),
+                watcherSearchTemplateService(), logger);
         ScheduleTriggerEvent event = new ScheduleTriggerEvent(watch.id(), DateTime.now(DateTimeZone.UTC), DateTime.now(DateTimeZone.UTC));
         Wid wid = new Wid("_record", randomLong(), DateTime.now(DateTimeZone.UTC));
         TriggeredWatch triggeredWatch = new TriggeredWatch(wid, event);
@@ -38,4 +40,8 @@ public class TriggeredWatchTests extends AbstractWatcherIntegrationTestCase {
     private TriggeredWatch.Parser triggeredWatchParser() {
         return internalCluster().getInstance(TriggeredWatch.Parser.class);
     }
+
+    protected WatcherSearchTemplateService watcherSearchTemplateService() {
+        return internalCluster().getInstance(WatcherSearchTemplateService.class);
+    }
 }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.rest.action;
 
-import com.google.common.collect.Lists;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -18,6 +17,8 @@ import org.elasticsearch.xpack.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.trigger.TriggerService;
 
+import java.util.Arrays;
+
 import static org.hamcrest.core.Is.is;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -32,9 +33,9 @@ public class RestExecuteWatchActionTests extends ESTestCase {
 
     public void testThatFlagsCanBeSpecifiedViaParameters() throws Exception {
         String randomId = randomAsciiOfLength(10);
-        for (String recordExecution : Lists.newArrayList("true", "false", null)) {
+        for (String recordExecution : Arrays.asList("true", "false", null)) {
-            for (String ignoreCondition : Lists.newArrayList("true", "false", null)) {
+            for (String ignoreCondition : Arrays.asList("true", "false", null)) {
-                for (String debugCondition : Lists.newArrayList("true", "false", null)) {
+                for (String debugCondition : Arrays.asList("true", "false", null)) {
                     ExecuteWatchRequestBuilder builder = new ExecuteWatchRequestBuilder(client);
                     when(watcherClient.prepareExecuteWatch()).thenReturn(builder);
 

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherUtilsTests.java

@@ -6,23 +6,6 @@
 package org.elasticsearch.xpack.watcher.support;
 
 
-import static java.util.Collections.singletonMap;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.formatDate;
-import static org.elasticsearch.xpack.watcher.support.WatcherUtils.DEFAULT_INDICES_OPTIONS;
-import static org.elasticsearch.xpack.watcher.support.WatcherUtils.flattenModel;
-import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType;
-import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.is;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -40,15 +23,30 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput;
 import org.elasticsearch.xpack.support.clock.SystemClock;
-import org.elasticsearch.xpack.common.text.TextTemplate;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.joda.time.DateTime;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput.DEFAULT_SEARCH_TYPE;
+import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.formatDate;
+import static org.elasticsearch.xpack.watcher.support.WatcherUtils.flattenModel;
+import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
 /**
  *
  */
@@ -100,6 +98,7 @@ public class WatcherUtilsTests extends ESTestCase {
     public void testSerializeSearchRequest() throws Exception {
         String[] randomIndices = generateRandomStringArray(5, 5, false);
         SearchRequest expectedRequest = new SearchRequest(randomIndices);
+        Script expectedTemplate = null;
 
         if (randomBoolean()) {
             String[] randomTypes = generateRandomStringArray(2, 5, false);
@@ -107,7 +106,7 @@ public class WatcherUtilsTests extends ESTestCase {
         }
 
         expectedRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(),
-                randomBoolean(), WatcherUtils.DEFAULT_INDICES_OPTIONS));
+                randomBoolean(), WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS));
         expectedRequest.searchType(getRandomSupportedSearchType());
 
         SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()).size(11);
@@ -122,30 +121,29 @@ public class WatcherUtilsTests extends ESTestCase {
                 }
             }
             String text = randomAsciiOfLengthBetween(1, 5);
-            Template template = randomFrom(
+            expectedTemplate = randomFrom(Script.inline(text), Script.file(text), Script.indexed(text)).params(params).build();
-                    new Template(text, ScriptType.INLINE, null, null, params),
-                    new Template(text, ScriptType.FILE, null, null, params),
-                    new Template(text, ScriptType.STORED, null, null, params)
-            );
-            expectedRequest.template(template);
         }
 
+        WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(expectedRequest, expectedTemplate);
+
         XContentBuilder builder = jsonBuilder();
-        builder = WatcherUtils.writeSearchRequest(expectedRequest, builder, ToXContent.EMPTY_PARAMS);
+        request.toXContent(builder, ToXContent.EMPTY_PARAMS);
         XContentParser parser = XContentHelper.createParser(builder.bytes());
         assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
         QueryParser<MatchAllQueryBuilder> queryParser = MatchAllQueryBuilder::fromXContent;
         registry.register(queryParser, MatchAllQueryBuilder.QUERY_NAME_FIELD);
         QueryParseContext context = new QueryParseContext(registry, parser, ParseFieldMatcher.STRICT);
-        SearchRequest result = WatcherUtils.readSearchRequest(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context, null, null);
+        WatcherSearchTemplateRequest result = WatcherSearchTemplateRequest.fromXContent(parser, DEFAULT_SEARCH_TYPE, context, null, null);
 
-        assertThat(result.indices(), arrayContainingInAnyOrder(expectedRequest.indices()));
+        assertThat(result.getRequest(), is(notNullValue()));
-        assertThat(result.types(), arrayContainingInAnyOrder(expectedRequest.types()));
+        assertThat(result.getRequest().indices(), arrayContainingInAnyOrder(expectedRequest.indices()));
-        assertThat(result.indicesOptions(), equalTo(expectedRequest.indicesOptions()));
+        assertThat(result.getRequest().types(), arrayContainingInAnyOrder(expectedRequest.types()));
-        assertThat(result.searchType(), equalTo(expectedRequest.searchType()));
+        assertThat(result.getRequest().indicesOptions(), equalTo(expectedRequest.indicesOptions()));
-        assertThat(result.source(), equalTo(searchSourceBuilder));
+        assertThat(result.getRequest().searchType(), equalTo(expectedRequest.searchType()));
-        assertThat(result.template(), equalTo(expectedRequest.template()));
+        assertThat(result.getRequest().source(), equalTo(searchSourceBuilder));
+
+        assertThat(result.getTemplate(), equalTo(expectedTemplate));
     }
 
     public void testDeserializeSearchRequest() throws Exception {
@@ -172,10 +170,10 @@ public class WatcherUtilsTests extends ESTestCase {
             }
         }
 
-        IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
+        IndicesOptions indicesOptions = WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS;
         if (randomBoolean()) {
             indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(),
-                    randomBoolean(), WatcherUtils.DEFAULT_INDICES_OPTIONS);
+                    randomBoolean(), WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS);
             builder.startObject("indices_options")
                     .field("allow_no_indices", indicesOptions.allowNoIndices())
                     .field("expand_wildcards", indicesOptions.expandWildcardsClosed() && indicesOptions.expandWildcardsOpen() ? "all" :
@@ -201,7 +199,7 @@ public class WatcherUtilsTests extends ESTestCase {
             source = searchSourceBuilder.buildAsBytes(XContentType.JSON);
             builder.rawField("body", source);
         }
-        Template templateSource = null;
+        Script template = null;
         if (randomBoolean()) {
             Map<String, Object> params = new HashMap<>();
             if (randomBoolean()) {
@@ -211,14 +209,8 @@ public class WatcherUtilsTests extends ESTestCase {
                 }
             }
             String text = randomAsciiOfLengthBetween(1, 5);
-            TextTemplate template = randomFrom(
+            template = randomFrom(Script.inline(text), Script.file(text), Script.indexed(text)) .params(params).build();
-                    TextTemplate.inline(text).params(params).build(),
-                    TextTemplate.file(text).params(params).build(),
-                    TextTemplate.indexed(text).params(params).build()
-            );
             builder.field("template", template);
-            templateSource = new Template(template.getTemplate(), template.getType(), null, template.getContentType(),
-                    template.getParams());
         }
         builder.endObject();
 
@@ -228,14 +220,15 @@ public class WatcherUtilsTests extends ESTestCase {
         QueryParser<MatchAllQueryBuilder> queryParser = MatchAllQueryBuilder::fromXContent;
         registry.register(queryParser, MatchAllQueryBuilder.QUERY_NAME_FIELD);
         QueryParseContext context = new QueryParseContext(registry, parser, ParseFieldMatcher.STRICT);
-        SearchRequest result = WatcherUtils.readSearchRequest(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context, null, null);
+        WatcherSearchTemplateRequest result = WatcherSearchTemplateRequest.fromXContent(parser, DEFAULT_SEARCH_TYPE, context, null, null);
 
-        assertThat(result.indices(), arrayContainingInAnyOrder(indices));
+        assertThat(result.getRequest(), is(notNullValue()));
-        assertThat(result.types(), arrayContainingInAnyOrder(types));
+        assertThat(result.getRequest().indices(), arrayContainingInAnyOrder(indices));
-        assertThat(result.indicesOptions(), equalTo(indicesOptions));
+        assertThat(result.getRequest().types(), arrayContainingInAnyOrder(types));
-        assertThat(result.searchType(), equalTo(searchType));
+        assertThat(result.getRequest().indicesOptions(), equalTo(indicesOptions));
-        assertThat(result.source(), equalTo(searchSourceBuilder));
+        assertThat(result.getRequest().searchType(), equalTo(searchType));
-        assertThat(result.template(), equalTo(templateSource));
+        assertThat(result.getRequest().source(), equalTo(searchSourceBuilder));
+        assertThat(result.getTemplate(), equalTo(template));
     }
 
 }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java

@@ -31,6 +31,13 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.common.ScriptServiceProxy;
+import org.elasticsearch.xpack.common.http.HttpClient;
+import org.elasticsearch.xpack.common.http.HttpMethod;
+import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
+import org.elasticsearch.xpack.common.secret.Secret;
+import org.elasticsearch.xpack.common.text.TextTemplate;
+import org.elasticsearch.xpack.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.notification.email.Authentication;
 import org.elasticsearch.xpack.notification.email.EmailService;
 import org.elasticsearch.xpack.notification.email.EmailTemplate;
@@ -49,15 +56,9 @@ import org.elasticsearch.xpack.watcher.execution.Wid;
 import org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput;
 import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
-import org.elasticsearch.xpack.common.ScriptServiceProxy;
-import org.elasticsearch.xpack.watcher.support.WatcherUtils;
-import org.elasticsearch.xpack.common.http.HttpClient;
-import org.elasticsearch.xpack.common.http.HttpMethod;
-import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
-import org.elasticsearch.xpack.common.secret.Secret;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
-import org.elasticsearch.xpack.common.text.TextTemplate;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
-import org.elasticsearch.xpack.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.support.xcontent.ObjectPath;
 import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.watcher.transform.search.ExecutableSearchTransform;
@@ -76,7 +77,6 @@ import javax.mail.internet.AddressException;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -122,7 +122,7 @@ public final class WatcherTestUtils {
     public static SearchRequest newInputSearchRequest(String... indices) {
         SearchRequest request = new SearchRequest();
         request.indices(indices);
-        request.indicesOptions(WatcherUtils.DEFAULT_INDICES_OPTIONS);
+        request.indicesOptions(WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS);
         request.searchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE);
         return request;
     }
@@ -175,13 +175,14 @@ public final class WatcherTestUtils {
 
 
     public static Watch createTestWatch(String watchName, HttpClient httpClient, EmailService emailService,
-                                        ESLogger logger) throws AddressException {
+                                        WatcherSearchTemplateService searchTemplateService, ESLogger logger) throws AddressException {
-        return createTestWatch(watchName, WatcherClientProxy.of(ESIntegTestCase.client()), httpClient, emailService, logger);
+        WatcherClientProxy client = WatcherClientProxy.of(ESIntegTestCase.client());
+        return createTestWatch(watchName, client, httpClient, emailService, searchTemplateService, logger);
     }
 
 
     public static Watch createTestWatch(String watchName, WatcherClientProxy client, HttpClient httpClient, EmailService emailService,
-                                        ESLogger logger) throws AddressException {
+                                        WatcherSearchTemplateService searchTemplateService, ESLogger logger) throws AddressException {
 
         SearchRequest conditionRequest = newInputSearchRequest("my-condition-index").source(searchSource().query(matchAllQuery()));
         SearchRequest transformRequest = newInputSearchRequest("my-payload-index").source(searchSource().query(matchAllQuery()));
@@ -229,12 +230,15 @@ public final class WatcherTestUtils {
         Map<String, ActionStatus> statuses = new HashMap<>();
         statuses.put("_webhook", new ActionStatus(now));
         statuses.put("_email", new ActionStatus(now));
+
+        SearchTransform searchTransform = new SearchTransform(new WatcherSearchTemplateRequest(transformRequest), null, null);
+
         return new Watch(
                 watchName,
                 new ScheduleTrigger(new CronSchedule("0/5 * * * * ? *")),
                 new ExecutableSimpleInput(new SimpleInput(new Payload.Simple(inputData)), logger),
                 new ExecutableAlwaysCondition(logger),
-                new ExecutableSearchTransform(new SearchTransform(transformRequest, null, null), logger, client, null),
+                new ExecutableSearchTransform(searchTransform, logger, client, searchTemplateService, null),
                 new TimeValue(0),
                 new ExecutableActions(actions),
                 metadata,
@@ -247,7 +251,7 @@ public final class WatcherTestUtils {
                 .put("script.indexed", "true")
                 .put("path.home", createTempDir())
                 .build();
-        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Arrays.asList(ScriptServiceProxy.INSTANCE));
+        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.singletonList(ScriptServiceProxy.INSTANCE));
 
         ScriptEngineRegistry scriptEngineRegistry =
                 new ScriptEngineRegistry(Collections.emptyList());

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java

@@ -7,20 +7,16 @@ package org.elasticsearch.xpack.watcher.test.integration;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.Callback;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.xpack.support.clock.SystemClock;
 import org.elasticsearch.xpack.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.watcher.condition.compare.CompareCondition;
-import org.elasticsearch.xpack.watcher.support.WatcherUtils;
+import org.elasticsearch.xpack.watcher.support.Script;
-import org.elasticsearch.xpack.support.clock.SystemClock;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.transport.actions.delete.DeleteWatchResponse;
@@ -151,8 +147,8 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
         watchSource.field("unknown_field", "x");
         watchSource.startObject("schedule").field("cron", "0/5 * * * * ? *").endObject();
 
-        watchSource.startObject("condition").startObject("script").field("script", "return true").field("request");
+        watchSource.startObject("condition").startObject("script").field("script", "return true");
-        WatcherUtils.writeSearchRequest(newInputSearchRequest(), watchSource, ToXContent.EMPTY_PARAMS);
+        watchSource.field("request", new WatcherSearchTemplateRequest(newInputSearchRequest()));
         watchSource.endObject().endObject();
 
         watchSource.endObject();
@@ -252,22 +248,20 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
 
     public void testConditionSearchWithSource() throws Exception {
         SearchSourceBuilder searchSourceBuilder = searchSource().query(matchQuery("level", "a"));
-        testConditionSearch(newInputSearchRequest("events").source(searchSourceBuilder));
+        testConditionSearch(newInputSearchRequest("events").source(searchSourceBuilder), null);
     }
 
     public void testConditionSearchWithIndexedTemplate() throws Exception {
         SearchSourceBuilder searchSourceBuilder = searchSource().query(matchQuery("level", "a"));
-        client().admin().cluster().preparePutStoredScript()
+        assertAcked(client().admin().cluster().preparePutStoredScript()
                 .setScriptLang("mustache")
                 .setId("my-template")
                 .setSource(jsonBuilder().startObject().field("template").value(searchSourceBuilder).endObject().bytes())
-                .get();
+                .get());
 
-        Template template = new Template("my-template", ScriptType.STORED, null, null, null);
+        Script template = Script.indexed("my-template").lang("mustache").build();
         SearchRequest searchRequest = newInputSearchRequest("events");
-        // TODO (2.0 upgrade): move back to BytesReference instead of coverting to a string
+        testConditionSearch(searchRequest, template);
-        searchRequest.template(template);
-        testConditionSearch(searchRequest);
     }
 
     public void testInputFiltering() throws Exception {
@@ -298,12 +292,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
 
         // Check that the input result payload has been filtered
         refresh();
-        SearchResponse searchResponse = searchWatchRecords(new Callback<SearchRequestBuilder>() {
+        SearchResponse searchResponse = searchWatchRecords(builder -> builder.setQuery(matchQuery("watch_id", "_name1")));
-            @Override
-            public void handle(SearchRequestBuilder builder) {
-                builder.setQuery(matchQuery("watch_id", "_name1"));
-            }
-        });
         assertHitCount(searchResponse, 1);
         XContentSource source = xContentSource(searchResponse.getHits().getAt(0).getSourceRef());
         assertThat(source.getValue("result.input.payload.hits.total"), equalTo((Object) 1));
@@ -379,17 +368,17 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
         }
     }
 
-    private void testConditionSearch(SearchRequest request) throws Exception {
+    private void testConditionSearch(SearchRequest request, Script template) throws Exception {
         // reset, so we don't miss event docs when we filter over the _timestamp field.
         timeWarp().clock().setTime(SystemClock.INSTANCE.nowUTC());
 
         String watchName = "_name";
-        assertAcked(prepareCreate("events").addMapping("event", "_timestamp", "enabled=true", "level", "type=text"));
+        assertAcked(prepareCreate("events").addMapping("event", "level", "type=text"));
 
         watcherClient().preparePutWatch(watchName)
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
-                        .input(searchInput(request))
+                        .input(searchInput(new WatcherSearchTemplateRequest(request, template)))
                         .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GTE, 3L)))
                 .get();
 

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatcherSettingsFilterTests.java

@@ -5,32 +5,29 @@
  */
 package org.elasticsearch.xpack.watcher.test.integration;
 
+import org.apache.http.Header;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.junit.After;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Map;
 
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-import static org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase.SecuritySettings.TEST_PASSWORD;
-import static org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase.SecuritySettings.TEST_USERNAME;
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.is;
 
-/**
- *
- */
 public class WatcherSettingsFilterTests extends AbstractWatcherIntegrationTestCase {
     private CloseableHttpClient httpClient = HttpClients.createDefault();
 
@@ -52,37 +49,26 @@ public class WatcherSettingsFilterTests extends AbstractWatcherIntegrationTestCa
     }
 
     public void testGetSettingsSmtpPassword() throws Exception {
-        String body = executeRequest("GET", "/_nodes/settings", null, null).getBody();
+        Header[] headers;
-        Map<String, Object> response = JsonXContent.jsonXContent.createParser(body).map();
+        if (securityEnabled()) {
-        Map<String, Object> nodes = (Map<String, Object>) response.get("nodes");
+            headers = new Header[] {
-        for (Object node : nodes.values()) {
+                    new BasicHeader(BASIC_AUTH_HEADER,
-            Map<String, Object> settings = (Map<String, Object>) ((Map<String, Object>) node).get("settings");
+                            basicAuthHeaderValue(MarvelIntegTestCase.SecuritySettings.TEST_USERNAME,
-            assertThat(XContentMapValues.extractValue("xpack.notification.email.account._email.smtp.user", settings),
+                                    new SecuredString(MarvelIntegTestCase.SecuritySettings.TEST_PASSWORD.toCharArray())))};
-                    is((Object) "_user"));
+        } else {
-            assertThat(XContentMapValues.extractValue("xpack.notification.email.account._email.smtp.password", settings),
+            headers = new Header[0];
-                    nullValue());
         }
-    }
+        try (Response response = getRestClient().performRequest("GET", "/_nodes/settings",
-
+                Collections.emptyMap(), null, headers)) {
-    protected HttpResponse executeRequest(String method, String path, String body, Map<String, String> params) throws IOException {
+            Map<String, Object> responseMap = JsonXContent.jsonXContent.createParser(response.getEntity().getContent()).map();
-        HttpServerTransport httpServerTransport = getInstanceFromMaster(HttpServerTransport.class);
+            Map<String, Object> nodes = (Map<String, Object>) responseMap.get("nodes");
-        HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient)
+            for (Object node : nodes.values()) {
-                .httpTransport(httpServerTransport)
+                Map<String, Object> settings = (Map<String, Object>) ((Map<String, Object>) node).get("settings");
-                .method(method)
+                assertThat(XContentMapValues.extractValue("xpack.notification.email.account._email.smtp.user", settings),
-                .path(path);
+                        is((Object) "_user"));
-
+                assertThat(XContentMapValues.extractValue("xpack.notification.email.account._email.smtp.password", settings),
-        if (params != null) {
+                        nullValue());
-            for (Map.Entry<String, String> entry : params.entrySet()) {
-                requestBuilder.addParam(entry.getKey(), entry.getValue());
             }
         }
-        if (body != null) {
-            requestBuilder.body(body);
-        }
-        if (securityEnabled()) {
-            requestBuilder.addHeader(BASIC_AUTH_HEADER,
-                    basicAuthHeaderValue(TEST_USERNAME, new SecuredString(TEST_PASSWORD.toCharArray())));
-        }
-        return requestBuilder.execute();
     }
 }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java

@@ -5,14 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.watch;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.ESLogger;
@@ -25,12 +17,31 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.common.ScriptServiceProxy;
+import org.elasticsearch.xpack.common.http.HttpClient;
+import org.elasticsearch.xpack.common.http.HttpMethod;
+import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
+import org.elasticsearch.xpack.common.http.auth.HttpAuthRegistry;
+import org.elasticsearch.xpack.common.http.auth.basic.BasicAuthFactory;
+import org.elasticsearch.xpack.common.secret.SecretService;
+import org.elasticsearch.xpack.common.text.TextTemplate;
+import org.elasticsearch.xpack.common.text.TextTemplateEngine;
+import org.elasticsearch.xpack.notification.email.DataAttachment;
+import org.elasticsearch.xpack.notification.email.EmailService;
+import org.elasticsearch.xpack.notification.email.EmailTemplate;
+import org.elasticsearch.xpack.notification.email.HtmlSanitizer;
+import org.elasticsearch.xpack.notification.email.Profile;
+import org.elasticsearch.xpack.notification.email.attachment.EmailAttachments;
+import org.elasticsearch.xpack.notification.email.attachment.EmailAttachmentsParser;
+import org.elasticsearch.xpack.support.clock.Clock;
+import org.elasticsearch.xpack.support.clock.ClockMock;
+import org.elasticsearch.xpack.support.clock.SystemClock;
+import org.elasticsearch.xpack.watcher.WatcherLicensee;
 import org.elasticsearch.xpack.watcher.actions.ActionFactory;
 import org.elasticsearch.xpack.watcher.actions.ActionRegistry;
 import org.elasticsearch.xpack.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.watcher.actions.ActionWrapper;
 import org.elasticsearch.xpack.watcher.actions.ExecutableActions;
-import org.elasticsearch.xpack.notification.email.DataAttachment;
 import org.elasticsearch.xpack.watcher.actions.email.EmailAction;
 import org.elasticsearch.xpack.watcher.actions.email.EmailActionFactory;
 import org.elasticsearch.xpack.watcher.actions.email.ExecutableEmailAction;
@@ -68,28 +79,15 @@ import org.elasticsearch.xpack.watcher.input.search.SearchInputFactory;
 import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInputFactory;
-import org.elasticsearch.xpack.watcher.WatcherLicensee;
 import org.elasticsearch.xpack.watcher.support.Script;
-import org.elasticsearch.xpack.watcher.support.WatcherUtils;
-import org.elasticsearch.xpack.support.clock.Clock;
-import org.elasticsearch.xpack.support.clock.ClockMock;
-import org.elasticsearch.xpack.support.clock.SystemClock;
-import org.elasticsearch.xpack.common.http.HttpClient;
-import org.elasticsearch.xpack.common.http.HttpMethod;
-import org.elasticsearch.xpack.common.http.HttpRequestTemplate;
-import org.elasticsearch.xpack.common.http.auth.HttpAuthRegistry;
-import org.elasticsearch.xpack.common.http.auth.basic.BasicAuthFactory;
-import org.elasticsearch.xpack.common.ScriptServiceProxy;
 import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy;
-import org.elasticsearch.xpack.common.secret.SecretService;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
-import org.elasticsearch.xpack.common.text.TextTemplate;
+import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
-import org.elasticsearch.xpack.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.test.WatcherTestUtils;
 import org.elasticsearch.xpack.watcher.transform.ExecutableTransform;
 import org.elasticsearch.xpack.watcher.transform.TransformFactory;
 import org.elasticsearch.xpack.watcher.transform.TransformRegistry;
 import org.elasticsearch.xpack.watcher.transform.chain.ChainTransform;
-import org.elasticsearch.xpack.watcher.transform.chain.ChainTransformFactory;
 import org.elasticsearch.xpack.watcher.transform.chain.ExecutableChainTransform;
 import org.elasticsearch.xpack.watcher.transform.script.ExecutableScriptTransform;
 import org.elasticsearch.xpack.watcher.transform.script.ScriptTransform;
@@ -116,21 +114,23 @@ import org.elasticsearch.xpack.watcher.trigger.schedule.support.Month;
 import org.elasticsearch.xpack.watcher.trigger.schedule.support.MonthTimes;
 import org.elasticsearch.xpack.watcher.trigger.schedule.support.WeekTimes;
 import org.elasticsearch.xpack.watcher.trigger.schedule.support.YearTimes;
-import org.elasticsearch.xpack.notification.email.EmailService;
-import org.elasticsearch.xpack.notification.email.EmailTemplate;
-import org.elasticsearch.xpack.notification.email.HtmlSanitizer;
-import org.elasticsearch.xpack.notification.email.Profile;
-import org.elasticsearch.xpack.notification.email.attachment.EmailAttachments;
-import org.elasticsearch.xpack.notification.email.attachment.EmailAttachmentsParser;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.Before;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonMap;
 import static java.util.Collections.unmodifiableMap;
-
 import static org.elasticsearch.xpack.watcher.input.InputBuilders.searchInput;
+import static org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS;
 import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.matchAllRequest;
 import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
 import static org.hamcrest.Matchers.equalTo;
@@ -153,6 +153,7 @@ public class WatchTests extends ESTestCase {
     private WatcherLicensee watcherLicensee;
     private ESLogger logger;
     private Settings settings = Settings.EMPTY;
+    private WatcherSearchTemplateService searchTemplateService;
 
     @Before
     public void init() throws Exception {
@@ -166,6 +167,7 @@ public class WatchTests extends ESTestCase {
         watcherLicensee = mock(WatcherLicensee.class);
         authRegistry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(secretService)));
         logger = Loggers.getLogger(WatchTests.class);
+        searchTemplateService = mock(WatcherSearchTemplateService.class);
     }
 
     public void testParserSelfGenerated() throws Exception {
@@ -341,7 +343,7 @@ public class WatchTests extends ESTestCase {
         switch (type) {
             case SearchInput.TYPE:
                 SearchInput searchInput = searchInput(WatcherTestUtils.newInputSearchRequest("idx")).build();
-                return new ExecutableSearchInput(searchInput, logger, client, null);
+                return new ExecutableSearchInput(searchInput, logger, client, searchTemplateService, null);
             default:
                 SimpleInput simpleInput = InputBuilders.simpleInput(singletonMap("_key", "_val")).build();
                 return new ExecutableSimpleInput(simpleInput, logger);
@@ -355,7 +357,7 @@ public class WatchTests extends ESTestCase {
                 IndicesQueriesRegistry queryRegistry = new IndicesQueriesRegistry();
                 QueryParser<MatchAllQueryBuilder> queryParser = MatchAllQueryBuilder::fromXContent;
                 queryRegistry.register(queryParser, MatchAllQueryBuilder.QUERY_NAME_FIELD);
-                parsers.put(SearchInput.TYPE, new SearchInputFactory(settings, client, queryRegistry, null, null));
+                parsers.put(SearchInput.TYPE, new SearchInputFactory(settings, client, queryRegistry, null, null, scriptService));
                 return new InputRegistry(Settings.EMPTY, parsers);
             default:
                 parsers.put(SimpleInput.TYPE, new SimpleInputFactory(settings));
@@ -406,15 +408,19 @@ public class WatchTests extends ESTestCase {
             case ScriptTransform.TYPE:
                 return new ExecutableScriptTransform(new ScriptTransform(Script.inline("_script").build()), logger, scriptService);
             case SearchTransform.TYPE:
-                return new ExecutableSearchTransform(new SearchTransform(
+                SearchTransform transform = new SearchTransform(
-                        matchAllRequest(WatcherUtils.DEFAULT_INDICES_OPTIONS), timeout, timeZone), logger, client, null);
+                        new WatcherSearchTemplateRequest(matchAllRequest(DEFAULT_INDICES_OPTIONS), null), timeout, timeZone);
+                return new ExecutableSearchTransform(transform, logger, client, searchTemplateService, null);
             default: // chain
-                ChainTransform chainTransform = new ChainTransform(Arrays.asList(
+                SearchTransform searchTransform = new SearchTransform(
-                        new SearchTransform(matchAllRequest(WatcherUtils.DEFAULT_INDICES_OPTIONS), timeout, timeZone),
+                        new WatcherSearchTemplateRequest(matchAllRequest(DEFAULT_INDICES_OPTIONS), null), timeout, timeZone);
-                        new ScriptTransform(Script.inline("_script").build())));
+                ScriptTransform scriptTransform = new ScriptTransform(Script.inline("_script").build());
+
+                ChainTransform chainTransform = new ChainTransform(Arrays.asList(searchTransform, scriptTransform));
                 return new ExecutableChainTransform(chainTransform, logger, Arrays.<ExecutableTransform>asList(
                         new ExecutableSearchTransform(new SearchTransform(
-                                matchAllRequest(WatcherUtils.DEFAULT_INDICES_OPTIONS), timeout, timeZone), logger, client, null),
+                                new WatcherSearchTemplateRequest(matchAllRequest(DEFAULT_INDICES_OPTIONS), null), timeout, timeZone),
+                                logger, client, searchTemplateService, null),
                         new ExecutableScriptTransform(new ScriptTransform(Script.inline("_script").build()), logger, scriptService)));
         }
     }
@@ -425,7 +431,7 @@ public class WatchTests extends ESTestCase {
         queryRegistry.register(queryParser, MatchAllQueryBuilder.QUERY_NAME_FIELD);
         Map<String, TransformFactory> factories = new HashMap<>();
         factories.put(ScriptTransform.TYPE, new ScriptTransformFactory(settings, scriptService));
-        factories.put(SearchTransform.TYPE, new SearchTransformFactory(settings, client, queryRegistry, null, null));
+        factories.put(SearchTransform.TYPE, new SearchTransformFactory(settings, client, queryRegistry, null, null, scriptService));
         TransformRegistry registry = new TransformRegistry(Settings.EMPTY, unmodifiableMap(factories));
         return registry;
     }

elasticsearch/x-pack/watcher/src/test/resources/rest-api-spec/api/xpack.watcher.ack_watch.json

@@ -4,7 +4,7 @@
     "methods": [ "PUT", "POST" ],
     "url": {
       "path": "/_xpack/watcher/watch/{watch_id}/_ack",
-      "paths": [ "/_xpack/watcher/watch/{watch_id}/_ack", "/_xpack/watcher/watch/{watch_id}/{action_id}/_ack"],
+      "paths": [ "/_xpack/watcher/watch/{watch_id}/_ack", "/_xpack/watcher/watch/{watch_id}/_ack/{action_id}"],
       "parts": {
         "watch_id": {
           "type" : "string",