diff --git a/elasticsearch/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java b/elasticsearch/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java index ba539f83908..97b0dd672f0 100644 --- a/elasticsearch/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java +++ b/elasticsearch/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java @@ -6,9 +6,11 @@ package org.elasticsearch.xpack.security.audit; import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.apache.http.message.BasicHeader; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Response; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.settings.Settings; @@ -19,7 +21,6 @@ import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.elasticsearch.xpack.XPackPlugin; import java.util.Collection; @@ -35,12 +36,12 @@ public class IndexAuditIT extends ESIntegTestCase { private static final String USER = "test_user"; private static final String PASS = "changeme"; - public void testIndexAuditTrailWorking() throws Exception { - HttpResponse response = httpClient().path("/") - .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(USER, new SecuredString(PASS.toCharArray()))) - .execute(); - assertThat(response.getStatusCode(), is(200)); - + public void testShieldIndexAuditTrailWorking() throws Exception { + try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(USER, new SecuredString(PASS.toCharArray()))))) { + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } final AtomicReference lastClusterState = new AtomicReference<>(); final AtomicBoolean indexExists = new AtomicBoolean(false); boolean found = awaitBusy(() -> { diff --git a/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyScriptConditionIT.java b/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyScriptConditionIT.java index 1830ee07723..8f9b29ff95a 100644 --- a/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyScriptConditionIT.java +++ b/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/GroovyScriptConditionIT.java @@ -66,13 +66,10 @@ public class GroovyScriptConditionIT extends AbstractWatcherIntegrationTestCase } public void testGroovyClosureWithAggregations() throws Exception { - client().admin().indices().prepareCreate(".monitoring") - .addMapping("cluster_stats", "_timestamp", "enabled=true") - .get(); - for (int seconds = 0; seconds < 60; seconds += 5) { - client().prepareIndex(".monitoring", "cluster_stats").setTimestamp("2005-01-01T00:00:" + - String.format(Locale.ROOT, "%02d", seconds)).setSource("status", randomFrom("green", "yellow")).get(); + String timestamp = "2005-01-01T00:00:" + String.format(Locale.ROOT, "%02d", seconds); + client().prepareIndex(".monitoring", "cluster_stats") + .setSource("status", randomFrom("green", "yellow"), "@timestamp", timestamp).get(); } refresh(); @@ -80,7 +77,7 @@ public class GroovyScriptConditionIT extends AbstractWatcherIntegrationTestCase SearchRequestBuilder builder = client().prepareSearch(".monitoring") .addAggregation( AggregationBuilders - .dateHistogram("minutes").field("_timestamp").interval(TimeUnit.MILLISECONDS.convert(5, TimeUnit.SECONDS)) + .dateHistogram("minutes").field("@timestamp").interval(TimeUnit.MILLISECONDS.convert(5, TimeUnit.SECONDS)) .order(Histogram.Order.COUNT_DESC) .subAggregation(AggregationBuilders.terms("status").field("status.keyword").size(3))); SearchResponse unmetResponse = builder.get(); @@ -101,8 +98,8 @@ public class GroovyScriptConditionIT extends AbstractWatcherIntegrationTestCase assertFalse(condition.execute(unmetContext).met()); for (int seconds = 0; seconds < 60; seconds += 5) { - client().prepareIndex(".monitoring", "cluster_stats").setTimestamp("2005-01-01T00:01:" + - String.format(Locale.ROOT, "%02d", seconds)).setSource("status", randomFrom("red")).get(); + String timestamp = "2005-01-01T00:01:" + String.format(Locale.ROOT, "%02d", seconds); + client().prepareIndex(".monitoring", "cluster_stats").setSource("status", randomFrom("red"), "@timestamp", timestamp).get(); } refresh(); diff --git a/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchIT.java b/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchIT.java index d467b68a68c..18c8b508cd0 100644 --- a/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchIT.java +++ b/elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/ScriptConditionSearchIT.java @@ -20,11 +20,11 @@ import org.elasticsearch.search.internal.InternalSearchHits; import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.common.ScriptServiceProxy; import org.elasticsearch.xpack.watcher.condition.script.ExecutableScriptCondition; import org.elasticsearch.xpack.watcher.condition.script.ScriptCondition; import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; import org.elasticsearch.xpack.watcher.support.Script; -import org.elasticsearch.xpack.common.ScriptServiceProxy; import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase; import org.elasticsearch.xpack.watcher.watch.Payload; import org.junit.After; @@ -61,18 +61,14 @@ public class ScriptConditionSearchIT extends AbstractWatcherIntegrationTestCase } public void testExecuteWithAggs() throws Exception { - client().admin().indices().prepareCreate("my-index") - .addMapping("my-type", "_timestamp", "enabled=true") - .get(); - - client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:00").setSource("{}").get(); - client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:10").setSource("{}").get(); - client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:20").setSource("{}").get(); - client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:30").setSource("{}").get(); + client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:00").get(); + client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:10").get(); + client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:20").get(); + client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:30").get(); refresh(); SearchResponse response = client().prepareSearch("my-index") - .addAggregation(AggregationBuilders.dateHistogram("rate").field("_timestamp") + .addAggregation(AggregationBuilders.dateHistogram("rate").field("@timestamp") .dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC)) .get(); @@ -83,10 +79,10 @@ public class ScriptConditionSearchIT extends AbstractWatcherIntegrationTestCase WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); assertFalse(condition.execute(ctx).met()); - client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:40").setSource("{}").get(); + client().prepareIndex("my-index", "my-type").setSource("@timestamp", "2005-01-01T00:40").get(); refresh(); - response = client().prepareSearch("my-index").addAggregation(AggregationBuilders.dateHistogram("rate").field("_timestamp") + response = client().prepareSearch("my-index").addAggregation(AggregationBuilders.dateHistogram("rate").field("@timestamp") .dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC)) .get(); diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java index b2d46d4a4ac..bb5a50d38d3 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java +++ b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java @@ -12,31 +12,40 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.Template; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.mustache.MustachePlugin; +import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.xpack.common.ScriptServiceProxy; import org.elasticsearch.xpack.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.actions.ActionWrapper; import org.elasticsearch.xpack.watcher.actions.ExecutableActions; import org.elasticsearch.xpack.watcher.condition.always.ExecutableAlwaysCondition; import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext; import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; +import org.elasticsearch.xpack.watcher.input.Input; import org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput; import org.elasticsearch.xpack.watcher.input.search.SearchInput; import org.elasticsearch.xpack.watcher.input.search.SearchInputFactory; import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput; import org.elasticsearch.xpack.watcher.input.simple.SimpleInput; +import org.elasticsearch.xpack.watcher.support.Script; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; +import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger; import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent; @@ -67,7 +76,6 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; import static org.joda.time.DateTimeZone.UTC; /** @@ -81,10 +89,11 @@ public class SearchInputIT extends ESIntegTestCase { Collection> types = new ArrayList<>(); types.addAll(super.nodePlugins()); types.add(MustachePlugin.class); + types.add(CustomScriptContextPlugin.class); return types; } - private final static String TEMPLATE_QUERY = "{\"query\":{\"filtered\":{\"query\":{\"match\":{\"event_type\":{\"query\":\"a\"," + + private final static String TEMPLATE_QUERY = "{\"query\":{\"bool\":{\"must\":{\"match\":{\"event_type\":{\"query\":\"a\"," + "\"type\":\"boolean\"}}},\"filter\":{\"range\":{\"_timestamp\":" + "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," + "\"include_lower\":true,\"include_upper\":true}}}}}}"; @@ -100,7 +109,6 @@ public class SearchInputIT extends ESIntegTestCase { throw new RuntimeException("failed to create config dir"); } - String path = "/org/elasticsearch/xpack/watcher/input/search/config/scripts/test_disk_template.mustache"; try (InputStream stream = SearchInputIT.class.getResourceAsStream("/org/elasticsearch/xpack/watcher/input/search/config/scripts" + "/test_disk_template.mustache"); OutputStream out = Files.newOutputStream(scriptPath.resolve("test_disk_template.mustache"))) { @@ -126,14 +134,15 @@ public class SearchInputIT extends ESIntegTestCase { SearchSourceBuilder searchSourceBuilder = searchSource().query( boolQuery().must(matchQuery("event_type", "a")).must(rangeQuery("_timestamp") .from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}"))); - SearchRequest request = client() + SearchRequest searchRequest = client() .prepareSearch() .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) .request() .source(searchSourceBuilder); + WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(searchRequest); ExecutableSearchInput searchInput = new ExecutableSearchInput(new SearchInput(request, null, null, null), logger, - WatcherClientProxy.of(client()), null); + WatcherClientProxy.of(client()), watcherSearchTemplateService(), null); WatchExecutionContext ctx = new TriggeredExecutionContext( new Watch("test-watch", new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))), @@ -149,21 +158,21 @@ public class SearchInputIT extends ESIntegTestCase { timeValueSeconds(5)); SearchInput.Result result = searchInput.execute(ctx, new Payload.Simple()); - assertThat((Integer) XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); + assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); assertNotNull(result.executedRequest()); - assertEquals(result.executedRequest().searchType(), request.searchType()); - assertArrayEquals(result.executedRequest().indices(), request.indices()); - assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions()); + assertThat(result.status(), is(Input.Result.Status.SUCCESS)); + assertEquals(result.executedRequest().searchType(), request.getRequest().searchType()); + assertArrayEquals(result.executedRequest().indices(), request.getRequest().indices()); + assertEquals(result.executedRequest().indicesOptions(), request.getRequest().indicesOptions()); + + XContentSource source = toXContentSource(result); + assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:00:00.000Z||-30s")); + assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:00:00.000Z")); } public void testSearchInlineTemplate() throws Exception { WatchExecutionContext ctx = createContext(); - final String expectedTemplateString = "{\"query\":{\"filtered\":{\"query\":{\"match\":{\"event_type\":{\"query\":\"a\"," - + "\"type\":\"boolean\"}}},\"filter\":{\"range\":{\"_timestamp\":" - + "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," - + "\"include_lower\":true,\"include_upper\":true}}}}}}"; - Map triggerParams = new HashMap(); triggerParams.put("triggered_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); triggerParams.put("scheduled_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); @@ -178,17 +187,28 @@ public class SearchInputIT extends ESIntegTestCase { Map expectedParams = new HashMap(); expectedParams.put("seconds_param", "30s"); expectedParams.put("ctx", ctxParams); - Template expectedTemplate = new Template(expectedTemplateString, ScriptType.INLINE, null, XContentType.JSON, expectedParams); Map params = new HashMap<>(); params.put("seconds_param", "30s"); - Template template = new Template(TEMPLATE_QUERY, ScriptType.INLINE, null, XContentType.JSON, params); + Script template = Script.inline(TEMPLATE_QUERY).lang("mustache").params(params).build(); - SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").setTemplate(template).request(); + SearchRequest request = client().prepareSearch() + .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) + .setIndices("test-search-index").request(); - SearchInput.Result executedResult = executeSearchInput(request, ctx); - assertThat(executedResult.executedRequest().template(), equalTo(expectedTemplate)); + SearchInput.Result executedResult = executeSearchInput(request, template, ctx); + + assertNotNull(executedResult.executedRequest()); + assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS)); + if (getNumShards("test-search-index").numPrimaries > 1) { + assertEquals(executedResult.executedRequest().searchType(), request.searchType()); + } + assertArrayEquals(executedResult.executedRequest().indices(), request.indices()); + assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions()); + + XContentSource source = toXContentSource(executedResult); + assertThat(source.getValue("query.bool.filter.0.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); + assertThat(source.getValue("query.bool.filter.0.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); } public void testSearchIndexedTemplate() throws Exception { @@ -204,17 +224,26 @@ public class SearchInputIT extends ESIntegTestCase { Map params = new HashMap<>(); params.put("seconds_param", "30s"); - Template template = new Template("test-template", ScriptType.STORED, null, null, params); + Script template = Script.indexed("test-template").lang("mustache").params(params).build(); jsonBuilder().value(TextTemplate.indexed("test-template").params(params).build()).bytes(); SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").setTemplate(template).request(); + .setIndices("test-search-index").request(); + + SearchInput.Result executedResult = executeSearchInput(request, template, ctx); + + assertNotNull(executedResult.executedRequest()); + assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS)); + if (getNumShards("test-search-index").numPrimaries > 1) { + assertEquals(executedResult.executedRequest().searchType(), request.searchType()); + } + assertArrayEquals(executedResult.executedRequest().indices(), request.indices()); + assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions()); + + XContentSource source = toXContentSource(executedResult); + assertThat(source.getValue("query.bool.filter.0.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); + assertThat(source.getValue("query.bool.filter.0.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); - SearchInput.Result executedResult = executeSearchInput(request, ctx); - Template resultTemplate = executedResult.executedRequest().template(); - assertThat(resultTemplate, notNullValue()); - assertThat(resultTemplate.getScript(), equalTo("test-template")); - assertThat(resultTemplate.getType(), equalTo(ScriptType.STORED)); } public void testSearchOnDiskTemplate() throws Exception { @@ -223,15 +252,16 @@ public class SearchInputIT extends ESIntegTestCase { Map params = new HashMap<>(); params.put("seconds_param", "30s"); - Template template = new Template("test_disk_template", ScriptType.FILE, null, null, params); + Script template = Script.file("test_disk_template").lang("mustache").params(params).build(); SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").setTemplate(template).request(); + .setIndices("test-search-index").request(); - SearchInput.Result executedResult = executeSearchInput(request, ctx); - Template resultTemplate = executedResult.executedRequest().template(); - assertThat(resultTemplate, notNullValue()); - assertThat(resultTemplate.getScript(), equalTo("test_disk_template")); - assertThat(resultTemplate.getType(), equalTo(ScriptType.FILE)); + SearchInput.Result executedResult = executeSearchInput(request, template, ctx); + + assertNotNull(executedResult.executedRequest()); + assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS)); + assertArrayEquals(executedResult.executedRequest().indices(), request.indices()); + assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions()); } public void testDifferentSearchType() throws Exception { @@ -241,14 +271,16 @@ public class SearchInputIT extends ESIntegTestCase { ); SearchType searchType = getRandomSupportedSearchType(); - SearchRequest request = client() + SearchRequest searchRequest = client() .prepareSearch() .setSearchType(searchType) .request() .source(searchSourceBuilder); + WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(searchRequest); + ExecutableSearchInput searchInput = new ExecutableSearchInput(new SearchInput(request, null, null, null), logger, - WatcherClientProxy.of(client()), null); + WatcherClientProxy.of(client()), watcherSearchTemplateService(), null); WatchExecutionContext ctx = new TriggeredExecutionContext( new Watch("test-watch", new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))), @@ -264,15 +296,20 @@ public class SearchInputIT extends ESIntegTestCase { timeValueSeconds(5)); SearchInput.Result result = searchInput.execute(ctx, new Payload.Simple()); - assertThat((Integer) XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); + assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); assertNotNull(result.executedRequest()); + assertThat(result.status(), is(Input.Result.Status.SUCCESS)); assertEquals(result.executedRequest().searchType(), searchType); - assertArrayEquals(result.executedRequest().indices(), request.indices()); - assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions()); + assertArrayEquals(result.executedRequest().indices(), searchRequest.indices()); + assertEquals(result.executedRequest().indicesOptions(), searchRequest.indicesOptions()); + + XContentSource source = toXContentSource(result); + assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:00:00.000Z||-30s")); + assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:00:00.000Z")); } public void testParserValid() throws Exception { - SearchRequest request = client().prepareSearch() + SearchRequest searchRequest = client().prepareSearch() .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) .request() .source(searchSource() @@ -280,13 +317,14 @@ public class SearchInputIT extends ESIntegTestCase { .from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}")))); TimeValue timeout = randomBoolean() ? TimeValue.timeValueSeconds(randomInt(10)) : null; - XContentBuilder builder = jsonBuilder().value(new SearchInput(request, null, timeout, null)); + XContentBuilder builder = jsonBuilder().value( + new SearchInput(new WatcherSearchTemplateRequest(searchRequest), null, timeout, null)); XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes()); parser.nextToken(); IndicesQueriesRegistry indicesQueryRegistry = internalCluster().getInstance(IndicesQueriesRegistry.class); SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, WatcherClientProxy.of(client()), indicesQueryRegistry, - null, null); + null, null, scriptService()); SearchInput searchInput = factory.parseInput("_id", parser); assertEquals(SearchInput.TYPE, searchInput.type()); @@ -309,15 +347,47 @@ public class SearchInputIT extends ESIntegTestCase { timeValueSeconds(5)); } - private SearchInput.Result executeSearchInput(SearchRequest request, WatchExecutionContext ctx) throws IOException { + private SearchInput.Result executeSearchInput(SearchRequest request, Script template, WatchExecutionContext ctx) throws IOException { createIndex("test-search-index"); ensureGreen("test-search-index"); - SearchInput.Builder siBuilder = SearchInput.builder(request); + SearchInput.Builder siBuilder = SearchInput.builder(new WatcherSearchTemplateRequest(request, template)); SearchInput si = siBuilder.build(); - ExecutableSearchInput searchInput = new ExecutableSearchInput(si, logger, WatcherClientProxy.of(client()), null); + ExecutableSearchInput searchInput = new ExecutableSearchInput(si, logger, WatcherClientProxy.of(client()), + watcherSearchTemplateService(), null); return searchInput.execute(ctx, new Payload.Simple()); } + protected WatcherSearchTemplateService watcherSearchTemplateService() { + String master = internalCluster().getMasterName(); + return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(), + ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master), internalCluster().clusterService(master)), + internalCluster().getInstance(IndicesQueriesRegistry.class, master), + internalCluster().getInstance(AggregatorParsers.class, master), + internalCluster().getInstance(Suggesters.class, master) + ); + } + + protected ScriptServiceProxy scriptService() { + return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class), internalCluster().clusterService()); + } + + private XContentSource toXContentSource(SearchInput.Result result) throws IOException { + try (XContentBuilder builder = jsonBuilder()) { + result.executedRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS); + return new XContentSource(builder); + } + } + + /** + * Custom plugin that registers XPack script context. + */ + public static class CustomScriptContextPlugin extends Plugin implements ScriptPlugin { + + @Override + public ScriptContext.Plugin getCustomScriptContexts() { + return ScriptServiceProxy.INSTANCE; + } + } } diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java index d2b9f47bbd3..7cb66e537fc 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java +++ b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java @@ -11,25 +11,28 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Requests; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.Template; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.mustache.MustachePlugin; +import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.xpack.common.ScriptServiceProxy; import org.elasticsearch.xpack.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.actions.ExecutableActions; import org.elasticsearch.xpack.watcher.condition.always.ExecutableAlwaysCondition; @@ -37,7 +40,11 @@ import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext; import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput; import org.elasticsearch.xpack.watcher.input.simple.SimpleInput; +import org.elasticsearch.xpack.watcher.support.Script; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; +import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.watcher.transform.Transform; import org.elasticsearch.xpack.watcher.transform.TransformBuilders; import org.elasticsearch.xpack.watcher.transform.search.ExecutableSearchTransform; @@ -100,6 +107,7 @@ public class SearchTransformIT extends ESIntegTestCase { Collection> types = new ArrayList<>(); types.addAll(super.nodePlugins()); types.add(MustachePlugin.class); + types.add(CustomScriptContextPlugin.class); return types; } @@ -154,7 +162,8 @@ public class SearchTransformIT extends ESIntegTestCase { SearchRequest request = Requests.searchRequest("idx").source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())); SearchTransform searchTransform = TransformBuilders.searchTransform(request).build(); - ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), null); + ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), + watcherSearchTemplateService(), null); WatchExecutionContext ctx = mockExecutionContext("_name", EMPTY_PAYLOAD); @@ -188,7 +197,8 @@ public class SearchTransformIT extends ESIntegTestCase { new SearchSourceBuilder().query(QueryBuilders.wrapperQuery(jsonBuilder().startObject() .startObject("_unknown_query_").endObject().endObject().bytes()))); SearchTransform searchTransform = TransformBuilders.searchTransform(request).build(); - ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), null); + ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), + watcherSearchTemplateService(), null); WatchExecutionContext ctx = mockExecutionContext("_name", EMPTY_PAYLOAD); @@ -200,23 +210,27 @@ public class SearchTransformIT extends ESIntegTestCase { assertThat(result.reason(), containsString("no [query] registered for [_unknown_query_]")); // extract the base64 encoded query from the template script, path is: query -> wrapper -> query - String jsonQuery = result.executedRequest().template().getScript(); - Map map = XContentFactory.xContent(jsonQuery).createParser(jsonQuery).map(); + try (XContentBuilder builder = jsonBuilder()) { + result.executedRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS); - assertThat(map, hasKey("query")); - assertThat(map.get("query"), instanceOf(Map.class)); + String jsonQuery = builder.string(); + Map map = XContentFactory.xContent(jsonQuery).createParser(jsonQuery).map(); - map = (Map) map.get("query"); - assertThat(map, hasKey("wrapper")); - assertThat(map.get("wrapper"), instanceOf(Map.class)); + assertThat(map, hasKey("query")); + assertThat(map.get("query"), instanceOf(Map.class)); - map = (Map) map.get("wrapper"); - assertThat(map, hasKey("query")); - assertThat(map.get("query"), instanceOf(String.class)); + map = (Map) map.get("query"); + assertThat(map, hasKey("wrapper")); + assertThat(map.get("wrapper"), instanceOf(Map.class)); - String queryAsBase64 = (String) map.get("query"); - String decodedQuery = new String(Base64.getDecoder().decode(queryAsBase64), StandardCharsets.UTF_8); - assertThat(decodedQuery, containsString("_unknown_query_")); + map = (Map) map.get("wrapper"); + assertThat(map, hasKey("query")); + assertThat(map.get("query"), instanceOf(String.class)); + + String queryAsBase64 = (String) map.get("query"); + String decodedQuery = new String(Base64.getDecoder().decode(queryAsBase64), StandardCharsets.UTF_8); + assertThat(decodedQuery, containsString("_unknown_query_")); + } } public void testExecuteMustacheTemplate() throws Exception { @@ -252,7 +266,8 @@ public class SearchTransformIT extends ESIntegTestCase { .must(termQuery("value", "{{ctx.payload.value}}")))); SearchTransform searchTransform = TransformBuilders.searchTransform(request).build(); - ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), null); + ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), + watcherSearchTemplateService(), null); ScheduleTriggerEvent event = new ScheduleTriggerEvent("_name", parseDate("2015-01-04T00:00:00", UTC), parseDate("2015-01-01T00:00:00", UTC)); @@ -319,24 +334,24 @@ public class SearchTransformIT extends ESIntegTestCase { IndicesQueriesRegistry indicesQueryRegistry = internalCluster().getInstance(IndicesQueriesRegistry.class); SearchTransformFactory transformFactory = new SearchTransformFactory(Settings.EMPTY, WatcherClientProxy.of(client()), - indicesQueryRegistry, null, null); + indicesQueryRegistry, null, null, scriptService()); ExecutableSearchTransform executable = transformFactory.parseExecutable("_id", parser); assertThat(executable, notNullValue()); assertThat(executable.type(), is(SearchTransform.TYPE)); assertThat(executable.transform().getRequest(), notNullValue()); if (indices != null) { - assertThat(executable.transform().getRequest().indices(), arrayContainingInAnyOrder(indices)); + assertThat(executable.transform().getRequest().getRequest().indices(), arrayContainingInAnyOrder(indices)); } if (searchType != null) { - assertThat(executable.transform().getRequest().searchType(), is(searchType)); + assertThat(executable.transform().getRequest().getRequest().searchType(), is(searchType)); } if (templateName != null) { - assertThat(executable.transform().getRequest().template(), - equalTo(new Template("template1", ScriptType.FILE, null, null, null))); + assertThat(executable.transform().getRequest().getTemplate(), + equalTo(Script.file("template1").build())); } SearchSourceBuilder source = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()); - assertThat(executable.transform().getRequest().source(), equalTo(source)); + assertThat(executable.transform().getRequest().getRequest().source(), equalTo(source)); assertThat(executable.transform().getTimeout(), equalTo(readTimeout)); } @@ -348,11 +363,6 @@ public class SearchTransformIT extends ESIntegTestCase { "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," + "\"include_lower\":true,\"include_upper\":true}}}]}}}"; - final String expectedTemplateString = "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"event_type\":{\"query\":\"a\"," - + "\"type\":\"boolean\"}}},{\"range\":{\"_timestamp\":" - + "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," - + "\"include_lower\":true,\"include_upper\":true}}}]}}}"; - Map triggerParams = new HashMap(); triggerParams.put("triggered_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); triggerParams.put("scheduled_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); @@ -367,18 +377,24 @@ public class SearchTransformIT extends ESIntegTestCase { Map expectedParams = new HashMap(); expectedParams.put("seconds_param", "30s"); expectedParams.put("ctx", ctxParams); - Template expectedTemplate = new Template(expectedTemplateString, ScriptType.INLINE, null, XContentType.JSON, expectedParams); Map params = new HashMap<>(); params.put("seconds_param", "30s"); - Template template = new Template(templateQuery, ScriptType.INLINE, null, XContentType.JSON, params); + Script template = Script.inline(templateQuery).lang("mustache").params(params).build(); SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").setTemplate(template).request(); + .setIndices("test-search-index").request(); - SearchTransform.Result executedResult = executeSearchTransform(request, ctx); + SearchTransform.Result executedResult = executeSearchTransform(request, template, ctx); - assertThat(executedResult.executedRequest().template(), equalTo(expectedTemplate)); + assertThat(executedResult.status(), is(Transform.Result.Status.SUCCESS)); + assertEquals(executedResult.executedRequest().searchType(), request.searchType()); + assertArrayEquals(executedResult.executedRequest().indices(), request.indices()); + assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions()); + + XContentSource source = toXContentSource(executedResult); + assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); + assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); } public void testSearchIndexedTemplate() throws Exception { @@ -399,24 +415,24 @@ public class SearchTransformIT extends ESIntegTestCase { Map params = new HashMap<>(); params.put("seconds_param", "30s"); - BytesReference templateSource = jsonBuilder() - .value(TextTemplate.indexed("test-script").params(params).build()) - .bytes(); - Template template = new Template("test-script", ScriptType.STORED, null, null, null); + Script template = Script.indexed("test-script").lang("mustache").params(params).build(); SearchRequest request = client() .prepareSearch() .setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE) .setIndices("test-search-index") - .setTemplate(template) .request(); - SearchTransform.Result result = executeSearchTransform(request, ctx); + SearchTransform.Result result = executeSearchTransform(request, template, ctx); + assertNotNull(result.executedRequest()); - Template resultTemplate = result.executedRequest().template(); - assertThat(resultTemplate, notNullValue()); - assertThat(resultTemplate.getScript(), equalTo("test-script")); - assertThat(resultTemplate.getType(), equalTo(ScriptType.STORED)); + assertThat(result.status(), is(Transform.Result.Status.SUCCESS)); + assertArrayEquals(result.executedRequest().indices(), request.indices()); + assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions()); + + XContentSource source = toXContentSource(result); + assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); + assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); } public void testSearchOnDiskTemplate() throws Exception { @@ -425,16 +441,20 @@ public class SearchTransformIT extends ESIntegTestCase { Map params = new HashMap<>(); params.put("seconds_param", "30s"); - Template template = new Template("test_disk_template", ScriptType.FILE, null, null, null); + Script template = Script.file("test_disk_template").lang("mustache").params(params).build(); SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").setTemplate(template).request(); + .setIndices("test-search-index").request(); + + SearchTransform.Result result = executeSearchTransform(request, template, ctx); - SearchTransform.Result result = executeSearchTransform(request, ctx); assertNotNull(result.executedRequest()); - Template resultTemplate = result.executedRequest().template(); - assertThat(resultTemplate, notNullValue()); - assertThat(resultTemplate.getScript(), equalTo("test_disk_template")); - assertThat(resultTemplate.getType(), equalTo(ScriptType.FILE)); + assertThat(result.status(), is(Transform.Result.Status.SUCCESS)); + assertArrayEquals(result.executedRequest().indices(), request.indices()); + assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions()); + + XContentSource source = toXContentSource(result); + assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); + assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); } public void testDifferentSearchType() throws Exception { @@ -453,13 +473,18 @@ public class SearchTransformIT extends ESIntegTestCase { .request() .source(searchSourceBuilder); - SearchTransform.Result result = executeSearchTransform(request, ctx); + SearchTransform.Result result = executeSearchTransform(request, null, ctx); - assertThat((Integer) XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); + assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); assertThat(result.executedRequest(), notNullValue()); + assertThat(result.status(), is(Transform.Result.Status.SUCCESS)); assertThat(result.executedRequest().searchType(), is(searchType)); assertThat(result.executedRequest().indices(), arrayContainingInAnyOrder(request.indices())); assertThat(result.executedRequest().indicesOptions(), equalTo(request.indicesOptions())); + + XContentSource source = toXContentSource(result); + assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); + assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); } private WatchExecutionContext createContext() { @@ -479,17 +504,31 @@ public class SearchTransformIT extends ESIntegTestCase { timeValueSeconds(5)); } - private SearchTransform.Result executeSearchTransform(SearchRequest request, WatchExecutionContext ctx) throws IOException { + private SearchTransform.Result executeSearchTransform(SearchRequest request, Script template, WatchExecutionContext ctx) + throws IOException { createIndex("test-search-index"); ensureGreen("test-search-index"); - SearchTransform searchTransform = TransformBuilders.searchTransform(request).build(); + SearchTransform searchTransform = TransformBuilders.searchTransform(new WatcherSearchTemplateRequest(request, template)).build(); ExecutableSearchTransform executableSearchTransform = new ExecutableSearchTransform(searchTransform, logger, - WatcherClientProxy.of(client()), null); + WatcherClientProxy.of(client()), watcherSearchTemplateService(), null); return executableSearchTransform.execute(ctx, Payload.Simple.EMPTY); } + protected WatcherSearchTemplateService watcherSearchTemplateService() { + String master = internalCluster().getMasterName(); + return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(), + ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class, master), internalCluster().clusterService(master)), + internalCluster().getInstance(IndicesQueriesRegistry.class, master), + internalCluster().getInstance(AggregatorParsers.class, master), + internalCluster().getInstance(Suggesters.class, master) + ); + } + + protected ScriptServiceProxy scriptService() { + return ScriptServiceProxy.of(internalCluster().getInstance(ScriptService.class), internalCluster().clusterService()); + } private static Map doc(String date, String value) { Map doc = new HashMap<>(); @@ -498,4 +537,21 @@ public class SearchTransformIT extends ESIntegTestCase { return doc; } + private XContentSource toXContentSource(SearchTransform.Result result) throws IOException { + try (XContentBuilder builder = jsonBuilder()) { + result.executedRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS); + return new XContentSource(builder); + } + } + + /** + * Custom plugin that registers XPack script context. + */ + public static class CustomScriptContextPlugin extends Plugin implements ScriptPlugin { + + @Override + public ScriptContext.Plugin getCustomScriptContexts() { + return ScriptServiceProxy.INSTANCE; + } + } } diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java index 048952045cd..99903025602 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java +++ b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java @@ -5,27 +5,27 @@ */ package org.elasticsearch.messy.tests; +import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.TermsLookup; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.Template; import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.script.mustache.MustacheScriptEngineService; -import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; +import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.junit.Before; import org.junit.BeforeClass; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; +import static java.util.Collections.singletonMap; +import static org.elasticsearch.script.ScriptService.ScriptType.INLINE; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; @@ -72,18 +72,6 @@ public class SecurityCachePermissionIT extends SecurityIntegTestCase { public void loadData() { index("data", "a", "1", "{ \"name\": \"John\", \"token\": \"token1\" }"); index("tokens", "tokens", "1", "{ \"group\": \"1\", \"tokens\": [\"token1\", \"token2\"] }"); - client().admin().cluster().preparePutStoredScript().setSource(new BytesArray("{\n" + - "\"template\": {\n" + - " \"query\": {\n" + - " \"exists\": {\n" + - " \"field\": \"{{name}}\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}")) - .setScriptLang("mustache") - .setId("testTemplate") - .execute().actionGet(); refresh(); } @@ -96,38 +84,44 @@ public class SecurityCachePermissionIT extends SecurityIntegTestCase { // Repeat with unauthorized user!!!! try { - response = client().filterWithHeader(Collections.singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER, + response = client().filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER, new SecuredString("changeme".toCharArray())))) .prepareSearch("data").setTypes("a").setQuery(QueryBuilders.constantScoreQuery( QueryBuilders.termsLookupQuery("token", new TermsLookup("tokens", "tokens", "1", "tokens")))) .execute().actionGet(); fail("search phase exception should have been thrown! response was:\n" + response.toString()); - } catch (SearchPhaseExecutionException e) { + } catch (ElasticsearchSecurityException e) { assertThat(e.toString(), containsString("ElasticsearchSecurityException[action")); assertThat(e.toString(), containsString("unauthorized")); } } public void testThatScriptServiceDoesntLeakData() { + String source = "{\n" + + "\"template\": {\n" + + " \"query\": {\n" + + " \"exists\": {\n" + + " \"field\": \"{{name}}\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + //Template template = new Template(source, INLINE, MustacheScriptEngineService.NAME, null, singletonMap("name", "token")); SearchResponse response = client().prepareSearch("data").setTypes("a") - .setTemplate(new Template("testTemplate", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null, - Collections.singletonMap("name", "token"))) + .setQuery(QueryBuilders.templateQuery(source, singletonMap("name", "token"))) .execute().actionGet(); assertThat(response.isTimedOut(), is(false)); assertThat(response.getHits().hits().length, is(1)); // Repeat with unauthorized user!!!! - try { - response = client().filterWithHeader(Collections.singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER, - new SecuredString("changeme".toCharArray())))) + ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> client() + .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER, + new SecuredString("changeme".toCharArray())))) .prepareSearch("data").setTypes("a") - .setTemplate(new Template("testTemplate", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null, - Collections.singletonMap("name", "token"))) - .execute().actionGet(); - fail("search phase exception should have been thrown! response was:\n" + response.toString()); - } catch (SearchPhaseExecutionException e) { - assertThat(e.toString(), containsString("ElasticsearchSecurityException[action")); - assertThat(e.toString(), containsString("unauthorized")); - } + .setQuery(QueryBuilders.templateQuery(source, singletonMap("name", "token"))) + .execute().actionGet()); + assertThat(e.toString(), containsString("ElasticsearchSecurityException[action")); + assertThat(e.toString(), containsString("unauthorized")); } } diff --git a/elasticsearch/qa/security-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java b/elasticsearch/qa/security-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java index 0e46268b2a2..2721747ea7a 100644 --- a/elasticsearch/qa/security-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java +++ b/elasticsearch/qa/security-example-realm/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java @@ -5,9 +5,12 @@ */ package org.elasticsearch.example.realm; +import org.apache.http.message.BasicHeader; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; @@ -16,7 +19,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.elasticsearch.xpack.XPackPlugin; import java.util.Collection; @@ -29,6 +31,7 @@ import static org.hamcrest.Matchers.is; * Integration test to test authentication with the custom realm */ public class CustomRealmIT extends ESIntegTestCase { + @Override protected Settings externalClusterClientSettings() { return Settings.builder() @@ -43,18 +46,23 @@ public class CustomRealmIT extends ESIntegTestCase { } public void testHttpConnectionWithNoAuthentication() throws Exception { - HttpResponse response = httpClient().path("/").execute(); - assertThat(response.getStatusCode(), is(401)); - String value = response.getHeaders().get("WWW-Authenticate"); - assertThat(value, is("custom-challenge")); + try { + getRestClient().performRequest("GET", "/", Collections.emptyMap(), null); + fail("request should have failed"); + } catch(ResponseException e) { + Response response = e.getResponse(); + assertThat(response.getStatusLine().getStatusCode(), is(401)); + String value = response.getHeader("WWW-Authenticate"); + assertThat(value, is("custom-challenge")); + } } public void testHttpAuthentication() throws Exception { - HttpResponse response = httpClient().path("/") - .addHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER) - .addHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW) - .execute(); - assertThat(response.getStatusCode(), is(200)); + try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null, + new BasicHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER), + new BasicHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW))) { + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } } public void testTransportClient() throws Exception { diff --git a/elasticsearch/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java b/elasticsearch/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java index ed7147b8edb..d4d92244c7c 100644 --- a/elasticsearch/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java +++ b/elasticsearch/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java @@ -7,9 +7,9 @@ package org.elasticsearch.smoketest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; + import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; @@ -19,6 +19,7 @@ import java.io.IOException; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + public class GraphWithSecurityIT extends ESRestTestCase { private final static String TEST_ADMIN_USERNAME = "test_admin"; diff --git a/elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java b/elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java index fa09330827d..5741fb72acc 100644 --- a/elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java +++ b/elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java @@ -14,7 +14,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; -import org.elasticsearch.test.rest.client.RestClient; +import org.elasticsearch.test.rest.client.RestTestClient; import org.elasticsearch.test.rest.parser.RestTestParseException; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -65,9 +65,9 @@ public class SmokeTestPluginsSslIT extends ESRestTestCase { String token = basicAuthHeaderValue(USER, new SecuredString(PASS.toCharArray())); return Settings.builder() .put(ThreadContext.PREFIX + ".Authorization", token) - .put(RestClient.PROTOCOL, "https") - .put(RestClient.TRUSTSTORE_PATH, keyStore) - .put(RestClient.TRUSTSTORE_PASSWORD, KEYSTORE_PASS) + .put(RestTestClient.PROTOCOL, "https") + .put(RestTestClient.TRUSTSTORE_PATH, keyStore) + .put(RestTestClient.TRUSTSTORE_PASSWORD, KEYSTORE_PASS) .build(); } } diff --git a/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java b/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java index 59430d0b378..af80d098c1c 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java +++ b/elasticsearch/qa/smoke-test-watcher-with-groovy/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java @@ -5,22 +5,19 @@ */ package org.elasticsearch.smoketest; -import java.io.IOException; -import java.net.URI; -import java.net.URL; - import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.BasicHttpClientConnectionManager; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import org.junit.After; import org.junit.Before; +import java.io.IOException; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + public abstract class WatcherRestTestCase extends ESRestTestCase { public WatcherRestTestCase(@Name("yaml") RestTestCandidate testCandidate) { @@ -34,19 +31,11 @@ public abstract class WatcherRestTestCase extends ESRestTestCase { @Before public void startWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_start", null, null)); - client.execute(request); - } + getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap()); } @After public void stopWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_stop", null, null)); - client.execute(request); - } + getAdminExecutionContext().callApi("xpack.watcher.stop", emptyMap(), emptyList(), emptyMap()); } } diff --git a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java index 17e0367317e..af80d098c1c 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java +++ b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java @@ -7,10 +7,6 @@ package org.elasticsearch.smoketest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.BasicHttpClientConnectionManager; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; @@ -18,8 +14,9 @@ import org.junit.After; import org.junit.Before; import java.io.IOException; -import java.net.URI; -import java.net.URL; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; public abstract class WatcherRestTestCase extends ESRestTestCase { @@ -34,19 +31,11 @@ public abstract class WatcherRestTestCase extends ESRestTestCase { @Before public void startWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_start", null, null)); - client.execute(request); - } + getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap()); } @After public void stopWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_stop", null, null)); - client.execute(request); - } + getAdminExecutionContext().callApi("xpack.watcher.stop", emptyMap(), emptyList(), emptyMap()); } } diff --git a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/10_webhook.yaml b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/10_webhook.yaml index b770ed31968..1a7ade5b783 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/10_webhook.yaml +++ b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/10_webhook.yaml @@ -55,7 +55,7 @@ catch: request_timeout cluster.health: wait_for_nodes: 99 - timeout: 10s + timeout: 5s - match: { "timed_out": true } - do: diff --git a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/20_array_access.yaml b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/20_array_access.yaml index 153d0334d42..bd051ab6d87 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/20_array_access.yaml +++ b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/20_array_access.yaml @@ -53,7 +53,7 @@ catch: request_timeout cluster.health: wait_for_nodes: 99 - timeout: 10s + timeout: 5s - match: { "timed_out": true } - do: diff --git a/elasticsearch/qa/smoke-test-watcher-with-painless/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java b/elasticsearch/qa/smoke-test-watcher-with-painless/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java index 59430d0b378..af80d098c1c 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-painless/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java +++ b/elasticsearch/qa/smoke-test-watcher-with-painless/src/test/java/org/elasticsearch/smoketest/WatcherRestTestCase.java @@ -5,22 +5,19 @@ */ package org.elasticsearch.smoketest; -import java.io.IOException; -import java.net.URI; -import java.net.URL; - import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.BasicHttpClientConnectionManager; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import org.junit.After; import org.junit.Before; +import java.io.IOException; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + public abstract class WatcherRestTestCase extends ESRestTestCase { public WatcherRestTestCase(@Name("yaml") RestTestCandidate testCandidate) { @@ -34,19 +31,11 @@ public abstract class WatcherRestTestCase extends ESRestTestCase { @Before public void startWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_start", null, null)); - client.execute(request); - } + getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap()); } @After public void stopWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_stop", null, null)); - client.execute(request); - } + getAdminExecutionContext().callApi("xpack.watcher.stop", emptyMap(), emptyList(), emptyMap()); } } diff --git a/elasticsearch/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/WatcherWithSecurityIT.java b/elasticsearch/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/WatcherWithSecurityIT.java index 8c141e45bfc..67ef62fcbaa 100644 --- a/elasticsearch/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/WatcherWithSecurityIT.java +++ b/elasticsearch/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/WatcherWithSecurityIT.java @@ -5,28 +5,26 @@ */ package org.elasticsearch.smoketest; -import java.io.IOException; -import java.net.URI; -import java.net.URL; - import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.BasicHttpClientConnectionManager; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xpack.security.authc.support.SecuredString; -import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import org.junit.After; import org.junit.Before; + +import java.io.IOException; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + public class WatcherWithSecurityIT extends ESRestTestCase { private final static String TEST_ADMIN_USERNAME = "test_admin"; @@ -43,24 +41,12 @@ public class WatcherWithSecurityIT extends ESRestTestCase { @Before public void startWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_start", null, null)); - String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecuredString(TEST_ADMIN_PASSWORD.toCharArray())); - request.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, token); - client.execute(request); - } + getAdminExecutionContext().callApi("xpack.watcher.start", emptyMap(), emptyList(), emptyMap()); } @After public void stopWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/watcher/_stop", null, null)); - String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecuredString(TEST_ADMIN_PASSWORD.toCharArray())); - request.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, token); - client.execute(request); - } + getAdminExecutionContext().callApi("xpack.watcher.stop", emptyMap(), emptyList(), emptyMap()); } @Override diff --git a/elasticsearch/x-pack/build.gradle b/elasticsearch/x-pack/build.gradle index e1f75abd90a..391172ce60b 100644 --- a/elasticsearch/x-pack/build.gradle +++ b/elasticsearch/x-pack/build.gradle @@ -29,7 +29,7 @@ dependencies { compile 'com.unboundid:unboundid-ldapsdk:2.3.8' compile 'org.bouncycastle:bcprov-jdk15on:1.54' compile 'org.bouncycastle:bcpkix-jdk15on:1.54' - testCompile 'com.google.jimfs:jimfs:1.0' + testCompile 'com.google.jimfs:jimfs:1.1' // watcher deps compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239' diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java index 6d69e5150aa..acccd2da2c9 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterStatsResolverTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.http.HttpInfo; import org.elasticsearch.index.Index; @@ -112,7 +113,7 @@ public class ClusterStatsResolverTests extends MonitoringIndexNameResolverTestCa Settings.EMPTY, DummyOsInfo.INSTANCE, new ProcessInfo(randomInt(), randomBoolean()), JvmInfo.jvmInfo(), new ThreadPoolInfo(Collections.singletonList(new ThreadPool.Info("test_threadpool", ThreadPool.ThreadPoolType.FIXED, 5))), new TransportInfo(transportAddress, Collections.emptyMap()), new HttpInfo(transportAddress, randomLong()), - new PluginsAndModules(), new IngestInfo(Collections.emptyList())); + new PluginsAndModules(), new IngestInfo(Collections.emptyList()), new ByteSizeValue(randomIntBetween(1, 1024))); } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java index 863e81fc0e6..3758a09d0c6 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/node/NodeStatsTests.java @@ -43,6 +43,7 @@ public class NodeStatsTests extends MarvelIntegTestCase { wipeMarvelIndices(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/2588") public void testNodeStats() throws Exception { logger.debug("--> creating some indices for future node stats"); final int numDocs = between(50, 150); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/security/MarvelSettingsFilterTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/security/MarvelSettingsFilterTests.java index c2f3f185324..0bc1db070d6 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/security/MarvelSettingsFilterTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/security/MarvelSettingsFilterTests.java @@ -5,21 +5,18 @@ */ package org.elasticsearch.marvel.security; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.Response; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.marvel.MonitoringSettings; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.xpack.security.authc.support.SecuredString; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.hamcrest.Matchers; -import org.junit.After; -import java.io.IOException; +import java.util.Collections; import java.util.Map; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; @@ -29,13 +26,6 @@ import static org.hamcrest.CoreMatchers.nullValue; public class MarvelSettingsFilterTests extends MarvelIntegTestCase { - private CloseableHttpClient httpClient = HttpClients.createDefault(); - - @After - public void cleanup() throws IOException { - httpClient.close(); - } - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -53,46 +43,35 @@ public class MarvelSettingsFilterTests extends MarvelIntegTestCase { } public void testGetSettingsFiltered() throws Exception { - String body = executeRequest("GET", "/_nodes/settings", null, null).getBody(); - Map response = JsonXContent.jsonXContent.createParser(body).map(); - Map nodes = (Map) response.get("nodes"); - for (Object node : nodes.values()) { - Map settings = (Map) ((Map) node).get("settings"); - - assertThat(extractValue("xpack.monitoring.agent.exporters._http.type", settings), Matchers.equalTo("http")); - assertThat(extractValue("xpack.monitoring.agent.exporters._http.enabled", settings), Matchers.equalTo("false")); - assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.username"); - assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.password"); - assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.path"); - assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.password"); - assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.hostname_verification"); + Header[] headers; + if (securityEnabled) { + headers = new Header[] { + new BasicHeader(BASIC_AUTH_HEADER, + basicAuthHeaderValue(SecuritySettings.TEST_USERNAME, + new SecuredString(SecuritySettings.TEST_PASSWORD.toCharArray())))}; + } else { + headers = new Header[0]; + } + try (Response response = getRestClient().performRequest("GET", "/_nodes/settings", + Collections.emptyMap(), null, headers)) { + Map responseMap = JsonXContent.jsonXContent.createParser(response.getEntity().getContent()).map(); + @SuppressWarnings("unchecked") + Map nodes = (Map) responseMap.get("nodes"); + for (Object node : nodes.values()) { + @SuppressWarnings("unchecked") + Map settings = (Map) ((Map) node).get("settings"); + assertThat(extractValue("xpack.monitoring.agent.exporters._http.type", settings), Matchers.equalTo("http")); + assertThat(extractValue("xpack.monitoring.agent.exporters._http.enabled", settings), Matchers.equalTo("false")); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.username"); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.auth.password"); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.path"); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.truststore.password"); + assertNullSetting(settings, "xpack.monitoring.agent.exporters._http.ssl.hostname_verification"); + } } } private void assertNullSetting(Map settings, String setting) { assertThat(extractValue(setting, settings), nullValue()); } - - protected HttpResponse executeRequest(String method, String path, String body, Map params) throws IOException { - HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class, - internalCluster().getMasterName()); - HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient) - .httpTransport(httpServerTransport) - .method(method) - .path(path); - - if (params != null) { - for (Map.Entry entry : params.entrySet()) { - requestBuilder.addParam(entry.getKey(), entry.getValue()); - } - } - if (body != null) { - requestBuilder.body(body); - } - if (securityEnabled) { - requestBuilder.addHeader(BASIC_AUTH_HEADER, - basicAuthHeaderValue(SecuritySettings.TEST_USERNAME, new SecuredString(SecuritySettings.TEST_PASSWORD.toCharArray()))); - } - return requestBuilder.execute(); - } } diff --git a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/RoleDescriptor.java b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/RoleDescriptor.java index 9ab3dd6f329..46cfa97c6aa 100644 --- a/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/RoleDescriptor.java +++ b/elasticsearch/x-pack/security/src/main/java/org/elasticsearch/xpack/security/authz/RoleDescriptor.java @@ -235,10 +235,6 @@ public class RoleDescriptor implements ToXContent { } } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.PRIVILEGES)) { privileges = readStringArray(roleName, parser, true); - if (names.length == 0) { - throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. [{}] cannot be an empty " + - "array", roleName, currentFieldName); - } } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Fields.FIELDS)) { fields = readStringArray(roleName, parser, true); } else { diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java index 2b98af55f07..01fc9f06b36 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java @@ -5,16 +5,18 @@ */ package org.elasticsearch.integration; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.elasticsearch.http.HttpServerTransport; +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.xpack.security.authc.support.Hasher; import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; -import org.junit.After; import java.io.IOException; import java.util.HashMap; @@ -32,19 +34,16 @@ public abstract class AbstractPrivilegeTestCase extends SecurityIntegTestCase { protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray()))); - private CloseableHttpClient httpClient = HttpClients.createDefault(); - - @After - public void cleanup() throws IOException { - httpClient.close(); - } - protected void assertAccessIsAllowed(String user, String method, String uri, String body, Map params) throws IOException { - HttpResponse response = executeRequest(user, method, uri, body, params); - String message = String.format(Locale.ROOT, "%s %s: Expected no error got %s %s with body %s", method, uri, - response.getStatusCode(), response.getReasonPhrase(), response.getBody()); - assertThat(message, response.getStatusCode(), is(not(greaterThanOrEqualTo(400)))); + try (Response response = getRestClient().performRequest(method, uri, params, entityOrNull(body), + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(user, new SecuredString("passwd".toCharArray()))))) { + StatusLine statusLine = response.getStatusLine(); + String message = String.format(Locale.ROOT, "%s %s: Expected no error got %s %s with body %s", method, uri, + statusLine.getStatusCode(), statusLine.getReasonPhrase(), EntityUtils.toString(response.getEntity())); + assertThat(message, statusLine.getStatusCode(), is(not(greaterThanOrEqualTo(400)))); + } } protected void assertAccessIsAllowed(String user, String method, String uri, String body) throws IOException { @@ -65,28 +64,24 @@ public abstract class AbstractPrivilegeTestCase extends SecurityIntegTestCase { protected void assertAccessIsDenied(String user, String method, String uri, String body, Map params) throws IOException { - HttpResponse response = executeRequest(user, method, uri, body, params); - String message = String.format(Locale.ROOT, "%s %s body %s: Expected 403, got %s %s with body %s", method, uri, body, - response.getStatusCode(), response.getReasonPhrase(), response.getBody()); - assertThat(message, response.getStatusCode(), is(403)); + try { + getRestClient().performRequest(method, uri, params, entityOrNull(body), + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(user, new SecuredString("passwd".toCharArray())))); + fail("request should have failed"); + } catch(ResponseException e) { + StatusLine statusLine = e.getResponse().getStatusLine(); + String message = String.format(Locale.ROOT, "%s %s body %s: Expected 403, got %s %s with body %s", method, uri, body, + statusLine.getStatusCode(), statusLine.getReasonPhrase(), e.getResponseBody()); + assertThat(message, statusLine.getStatusCode(), is(403)); + } } - protected HttpResponse executeRequest(String user, String method, String uri, String body, - Map params) throws IOException { - HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class); - - HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport); - requestBuilder.path(uri); - requestBuilder.method(method); - for (Map.Entry entry : params.entrySet()) { - requestBuilder.addParam(entry.getKey(), entry.getValue()); - } + private static HttpEntity entityOrNull(String body) { + HttpEntity entity = null; if (body != null) { - requestBuilder.body(body); + entity = new StringEntity(body, RestClient.JSON_CONTENT_TYPE); } - requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(user, - new SecuredString("passwd".toCharArray()))); - return requestBuilder.execute(); + return entity; } - } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java index 4114e171200..cda5beff25f 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java @@ -5,9 +5,15 @@ */ package org.elasticsearch.integration; +import org.apache.http.Header; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.security.Security; @@ -15,10 +21,10 @@ import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.elasticsearch.xpack.XPackPlugin; import java.io.IOException; +import java.util.Collections; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -49,8 +55,8 @@ public class BulkUpdateTests extends SecurityIntegTestCase { .get().isCreated(); assertThat(created, is(false)); getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1").setFields("test", "not test").get(); - assertThat((String) getResponse.getField("test").getValue(), equalTo("test")); - assertThat((String) getResponse.getField("not test").getValue(), equalTo("not test")); + assertThat(getResponse.getField("test").getValue(), equalTo("test")); + assertThat(getResponse.getField("not test").getValue(), equalTo("not test")); // this part is important. Without this, the document may be read from the translog which would bypass the bug where // FLS kicks in because the request can't be found and only returns meta fields @@ -62,43 +68,61 @@ public class BulkUpdateTests extends SecurityIntegTestCase { assertThat(((UpdateResponse)response.getItems()[0].getResponse()).isCreated(), is(false)); getResponse = internalCluster().transportClient().prepareGet("index1", "type", "1"). setFields("test", "not test", "bulk updated").get(); - assertThat((String) getResponse.getField("test").getValue(), equalTo("test")); - assertThat((String) getResponse.getField("not test").getValue(), equalTo("not test")); - assertThat((String) getResponse.getField("bulk updated").getValue(), equalTo("bulk updated")); + assertThat(getResponse.getField("test").getValue(), equalTo("test")); + assertThat(getResponse.getField("not test").getValue(), equalTo("not test")); + assertThat(getResponse.getField("bulk updated").getValue(), equalTo("bulk updated")); } public void testThatBulkUpdateDoesNotLoseFieldsHttp() throws IOException { final String path = "/index1/type/1"; - final String basicAuthHeader = UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, - new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray())); + final Header basicAuthHeader = new BasicHeader("Authorization", + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, + new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))); - httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("PUT").body("{\"test\":\"test\"}").execute(); - HttpResponse response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute(); - assertThat(response.getBody(), containsString("\"test\":\"test\"")); + StringEntity body = new StringEntity("{\"test\":\"test\"}", RestClient.JSON_CONTENT_TYPE); + try (Response response = getRestClient().performRequest("PUT", path, Collections.emptyMap(), body, basicAuthHeader)) { + assertThat(response.getStatusLine().getStatusCode(), equalTo(201)); + } + + try (Response response = getRestClient().performRequest("GET", path, Collections.emptyMap(), null, basicAuthHeader)) { + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(EntityUtils.toString(response.getEntity()), containsString("\"test\":\"test\"")); + } if (randomBoolean()) { flushAndRefresh(); } //update with new field - httpClient().path(path + "/_update").addHeader("Authorization", basicAuthHeader).method("POST"). - body("{\"doc\": {\"not test\": \"not test\"}}").execute(); - response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute(); - assertThat(response.getBody(), containsString("\"test\":\"test\"")); - assertThat(response.getBody(), containsString("\"not test\":\"not test\"")); + body = new StringEntity("{\"doc\": {\"not test\": \"not test\"}}", RestClient.JSON_CONTENT_TYPE); + try (Response response = getRestClient().performRequest("POST", path + "/_update", + Collections.emptyMap(), body, basicAuthHeader)) { + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + } + + try (Response response = getRestClient().performRequest("GET", path, Collections.emptyMap(), null, basicAuthHeader)) { + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseBody = EntityUtils.toString(response.getEntity()); + assertThat(responseBody, containsString("\"test\":\"test\"")); + assertThat(responseBody, containsString("\"not test\":\"not test\"")); + } // this part is important. Without this, the document may be read from the translog which would bypass the bug where // FLS kicks in because the request can't be found and only returns meta fields flushAndRefresh(); - // update with bulk - httpClient().path("/_bulk").addHeader("Authorization", basicAuthHeader).method("POST") - .body("{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n{\"doc\": {\"bulk updated\":\"bulk " + - "updated\"}}\n") - .execute(); - response = httpClient().path(path).addHeader("Authorization", basicAuthHeader).method("GET").execute(); - assertThat(response.getBody(), containsString("\"test\":\"test\"")); - assertThat(response.getBody(), containsString("\"not test\":\"not test\"")); - assertThat(response.getBody(), containsString("\"bulk updated\":\"bulk updated\"")); + body = new StringEntity("{\"update\": {\"_index\": \"index1\", \"_type\": \"type\", \"_id\": \"1\"}}\n" + + "{\"doc\": {\"bulk updated\":\"bulk updated\"}}\n", RestClient.JSON_CONTENT_TYPE); + try (Response response = getRestClient().performRequest("POST", "/_bulk", + Collections.emptyMap(), body, basicAuthHeader)) { + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + } + + try (Response response = getRestClient().performRequest("GET", path, Collections.emptyMap(), null, basicAuthHeader)) { + String responseBody = EntityUtils.toString(response.getEntity()); + assertThat(responseBody, containsString("\"test\":\"test\"")); + assertThat(responseBody, containsString("\"not test\":\"not test\"")); + assertThat(responseBody, containsString("\"bulk updated\":\"bulk updated\"")); + } } } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java index aacc4444afd..a5e9d01bc35 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java @@ -5,14 +5,15 @@ */ package org.elasticsearch.integration; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.http.HttpServerTransport; -import org.elasticsearch.xpack.security.user.User; +import org.elasticsearch.test.SecurityIntegTestCase; +import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheRequest; import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheResponse; import org.elasticsearch.xpack.security.authc.Realm; @@ -22,10 +23,7 @@ import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.security.authc.support.SecuredStringTests; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.security.client.SecurityClient; -import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; +import org.elasticsearch.xpack.security.user.User; import org.junit.BeforeClass; import java.util.ArrayList; @@ -39,14 +37,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; -/** - * - */ public class ClearRealmsCacheTests extends SecurityIntegTestCase { private static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("passwd".toCharArray()))); @@ -168,21 +162,12 @@ public class ClearRealmsCacheTests extends SecurityIntegTestCase { } static void executeHttpRequest(String path, Map params) throws Exception { - try (CloseableHttpClient client = HttpClients.createDefault()) { - HttpRequestBuilder requestBuilder = new HttpRequestBuilder(client) - .httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class)) - .method("POST") - .path(path); - for (Map.Entry entry : params.entrySet()) { - requestBuilder.addParam(entry.getKey(), entry.getValue()); - } - requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, - UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, - new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))); - HttpResponse response = requestBuilder.execute(); - assertThat(response.hasBody(), is(true)); - String body = response.getBody(); - assertThat(body.contains("cluster_name"), is(true)); + try (Response response = getRestClient().performRequest("POST", path, params, null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, + new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) { + assertNotNull(response.getEntity()); + assertTrue(EntityUtils.toString(response.getEntity()).contains("cluster_name")); } } } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java index bec5a581aa3..a9eca134b68 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java @@ -5,9 +5,11 @@ */ package org.elasticsearch.integration; +import org.apache.http.message.BasicHeader; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; @@ -23,11 +25,11 @@ import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.elasticsearch.xpack.security.client.SecurityClient; import org.elasticsearch.test.NativeRealmIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.junit.Before; import org.junit.BeforeClass; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -81,7 +83,7 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase { logger.debug("using poller interval [{}]", pollerInterval); return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(NativeRolesStore.POLL_INTERVAL_SETTING.getKey(), pollerInterval) + .put(NativeRolesStore.POLL_INTERVAL_SETTING.getKey(), pollerInterval.getStringRep()) .put(NetworkModule.HTTP_ENABLED.getKey(), true) .build(); } @@ -128,7 +130,7 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase { final boolean useHttp = randomBoolean(); final boolean clearAll = randomBoolean(); logger.debug("--> starting to clear roles. using http [{}] clearing all [{}]", useHttp, clearAll); - String[] rolesToClear = clearAll ? (randomBoolean() ? roles : null) : toModify.toArray(Strings.EMPTY_ARRAY); + String[] rolesToClear = clearAll ? (randomBoolean() ? roles : null) : toModify.toArray(new String[toModify.size()]); if (useHttp) { String path; if (rolesToClear == null) { @@ -136,12 +138,12 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase { } else { path = "/_xpack/security/role/" + Strings.arrayToCommaDelimitedString(rolesToClear) + "/_clear_cache"; } - HttpResponse response = httpClient().path(path).method("POST") - .addHeader("Authorization", + try (Response response = getRestClient().performRequest("POST", path, Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, - new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))) - .execute(); - assertThat(response.getStatusCode(), is(RestStatus.OK.getStatus())); + new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) { + assertThat(response.getStatusLine().getStatusCode(), is(RestStatus.OK.getStatus())); + } } else { securityClient.prepareClearRolesCache().names(rolesToClear).get(); } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java index 5519eea89fa..1bcd3b5042d 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java @@ -5,20 +5,21 @@ */ package org.elasticsearch.integration; -import org.apache.lucene.util.LuceneTestCase.BadApple; +import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.rest.client.http.HttpResponse; +import org.elasticsearch.xpack.security.authc.support.SecuredString; +import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.junit.Before; import java.util.Collections; -import java.util.HashMap; import java.util.Locale; import java.util.Map; import static java.util.Collections.singletonMap; -import static org.hamcrest.Matchers.containsString; +import static org.apache.lucene.util.LuceneTestCase.BadApple; import static org.hamcrest.Matchers.is; //test is just too slow, please fix it to not be sleep-based @@ -303,8 +304,14 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { } public void testThatUnknownUserIsRejectedProperly() throws Exception { - HttpResponse response = executeRequest("idonotexist", "GET", "/", null, new HashMap<>()); - assertThat(response.getStatusCode(), is(401)); + try { + getRestClient().performRequest("GET", "/", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue("idonotexist", new SecuredString("passwd".toCharArray())))); + fail("request should have failed"); + } catch(ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401)); + } } private void assertUserExecutes(String user, String action, String index, boolean userIsAllowed) throws Exception { diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/LicensingTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/LicensingTests.java index ab82c4f4ed5..4bb04f52720 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/LicensingTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/LicensingTests.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.component.AbstractComponent; @@ -194,13 +196,20 @@ public class LicensingTests extends SecurityIntegTestCase { } public void testRestAuthenticationByLicenseType() throws Exception { - // the default of the licensing tests is basic - assertThat(httpClient().path("/").execute().getStatusCode(), is(200)); + try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) { + // the default of the licensing tests is basic + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } // generate a new license with a mode that enables auth OperationMode mode = randomFrom(OperationMode.GOLD, OperationMode.TRIAL, OperationMode.PLATINUM, OperationMode.STANDARD); enableLicensing(mode); - assertThat(httpClient().path("/").execute().getStatusCode(), is(401)); + try { + getRestClient().performRequest("GET", "/", Collections.emptyMap(), null); + fail("request should have failed"); + } catch(ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401)); + } } public void testTransportClientAuthenticationByLicenseType() throws Exception { diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginEnabledDisabledTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginEnabledDisabledTests.java index 7d93416c63e..b2e9af0013a 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginEnabledDisabledTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginEnabledDisabledTests.java @@ -19,7 +19,6 @@ import org.junit.After; import org.junit.BeforeClass; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; /** diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java index 6778430cf8f..9c800857517 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/SecurityPluginTests.java @@ -5,23 +5,21 @@ */ package org.elasticsearch.xpack.security; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; import java.io.IOException; +import java.util.Collections; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.is; public class SecurityPluginTests extends SecurityIntegTestCase { @@ -35,24 +33,20 @@ public class SecurityPluginTests extends SecurityIntegTestCase { } public void testThatPluginIsLoaded() throws IOException { - HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class); - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + try { logger.info("executing unauthorized request to /_xpack info"); - HttpResponse response = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport) - .method("GET") - .path("/_xpack") - .execute(); - assertThat(response.getStatusCode(), is(UNAUTHORIZED.getStatus())); + getRestClient().performRequest("GET", "/_xpack", Collections.emptyMap(), null); + fail("request should have failed"); + } catch(ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(UNAUTHORIZED.getStatus())); + } - logger.info("executing authorized request to /_xpack infos"); - response = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport) - .method("GET") - .path("/_xpack") - .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + logger.info("executing authorized request to /_xpack infos"); + try (Response response = getRestClient().performRequest("GET", "/_xpack", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, - new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))) - .execute(); - assertThat(response.getStatusCode(), is(OK.getStatus())); + new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) { + assertThat(response.getStatusLine().getStatusCode(), is(OK.getStatus())); } } } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java index 0a62c5132e4..21cdc518047 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java @@ -5,10 +5,13 @@ */ package org.elasticsearch.xpack.security.authc; +import org.apache.http.message.BasicHeader; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; @@ -20,7 +23,6 @@ import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.security.transport.netty.SecurityNettyTransport; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.elasticsearch.xpack.XPackPlugin; import java.util.Collections; @@ -115,30 +117,36 @@ public class RunAsIntegTests extends SecurityIntegTestCase { public void testUserImpersonationUsingHttp() throws Exception { // use the transport client user and try to run as - HttpResponse response = httpClient().method("GET") - .path("/_nodes") - .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER, - SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))) - .addHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, SecuritySettingsSource.DEFAULT_USER_NAME) - .execute(); - assertThat(response.getStatusCode(), is(403)); + try { + getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER, + SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))), + new BasicHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, SecuritySettingsSource.DEFAULT_USER_NAME)); + fail("request should have failed"); + } catch(ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); + } - //the run as user shouldn't have access to the nodes api - response = httpClient().method("GET") - .path("/_nodes") - .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, - SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))) - .execute(); - assertThat(response.getStatusCode(), is(403)); + try { + //the run as user shouldn't have access to the nodes api + getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, + SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD)))); + fail("request should have failed"); + } catch(ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); + } // but when running as a different user it should work - response = httpClient().method("GET") - .path("/_nodes") - .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, - SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))) - .addHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, SecuritySettingsSource.DEFAULT_USER_NAME) - .execute(); - assertThat(response.getStatusCode(), is(200)); + try (Response response = getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, + SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))), + new BasicHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, SecuritySettingsSource.DEFAULT_USER_NAME))) { + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } } public void testEmptyUserImpersonationHeader() throws Exception { @@ -164,13 +172,16 @@ public class RunAsIntegTests extends SecurityIntegTestCase { } public void testEmptyHeaderUsingHttp() throws Exception { - HttpResponse response = httpClient().method("GET") - .path("/_nodes") - .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, - SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))) - .addHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, "") - .execute(); - assertThat(response.getStatusCode(), is(401)); + try { + getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, + SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))), + new BasicHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, "")); + fail("request should have failed"); + } catch(ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401)); + } } public void testNonExistentRunAsUser() throws Exception { @@ -196,13 +207,16 @@ public class RunAsIntegTests extends SecurityIntegTestCase { } public void testNonExistentRunAsUserUsingHttp() throws Exception { - HttpResponse response = httpClient().method("GET") - .path("/_nodes") - .addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, - SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))) - .addHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, "idontexist") - .execute(); - assertThat(response.getStatusCode(), is(403)); + try { + getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(RUN_AS_USER, + SecuredStringTests.build(SecuritySettingsSource.DEFAULT_PASSWORD))), + new BasicHeader(InternalAuthenticationService.RUN_AS_USER_HEADER, "idontexist")); + fail("request should have failed"); + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403)); + } } // build our own here to better mimic an actual client... diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java index 2be1bd68846..ad3d3d22964 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiOptionalClientAuthTests.java @@ -7,11 +7,14 @@ package org.elasticsearch.xpack.security.authc.pki; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; @@ -20,8 +23,6 @@ import org.elasticsearch.xpack.security.transport.netty.SecurityNettyHttpServerT import org.elasticsearch.xpack.security.transport.netty.SecurityNettyTransport; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.elasticsearch.transport.Transport; import org.elasticsearch.xpack.XPackPlugin; import org.junit.BeforeClass; @@ -34,6 +35,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyStore; import java.security.SecureRandom; +import java.util.Collections; import static org.elasticsearch.test.SecuritySettingsSource.DEFAULT_PASSWORD; import static org.elasticsearch.test.SecuritySettingsSource.DEFAULT_USER_NAME; @@ -78,23 +80,21 @@ public class PkiOptionalClientAuthTests extends SecurityIntegTestCase { } public void testRestClientWithoutClientCertificate() throws Exception { - HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class); + CloseableHttpClient httpClient = HttpClients.custom().setSSLContext(getSSLContext()).build(); + try (RestClient restClient = createRestClient(httpClient, "https")) { + try { + restClient.performRequest("GET", "_nodes", Collections.emptyMap(), null); + fail("request should have failed"); + } catch(ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401)); + } - try (CloseableHttpClient httpClient = HttpClients.custom().setSslcontext(getSSLContext()).build()) { - HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient) - .host("localhost") - .port(((InetSocketTransportAddress)httpServerTransport.boundAddress().publishAddress()).address().getPort()) - .protocol("https") - .method("GET") - .path("/_nodes"); - HttpResponse response = requestBuilder.execute(); - assertThat(response.getStatusCode(), is(401)); - - requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, - UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, - new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))); - response = requestBuilder.execute(); - assertThat(response.getStatusCode(), is(200)); + try (Response response = restClient.performRequest("GET", "_nodes", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, + new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) { + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } } } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutClientAuthenticationTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutClientAuthenticationTests.java index 6b16ef3fc75..6be44408b81 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutClientAuthenticationTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutClientAuthenticationTests.java @@ -8,27 +8,27 @@ package org.elasticsearch.xpack.security.authc.pki; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.SecurityIntegTestCase; +import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.security.transport.SSLClientAuth; import org.elasticsearch.xpack.security.transport.netty.SecurityNettyHttpServerTransport; import org.elasticsearch.xpack.security.transport.netty.SecurityNettyTransport; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import java.security.SecureRandom; import java.security.cert.X509Certificate; +import java.util.Collections; import java.util.Locale; import static org.hamcrest.Matchers.is; @@ -77,21 +77,16 @@ public class PkiWithoutClientAuthenticationTests extends SecurityIntegTestCase { } public void testThatHttpWorks() throws Exception { - HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class); SSLContext sc = SSLContext.getInstance("SSL"); sc.init(null, trustAllCerts, new SecureRandom()); - try (CloseableHttpClient httpClient = HttpClients.custom().setSslcontext(sc).build()) { - HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient) - .host("localhost") - .port(((InetSocketTransportAddress)httpServerTransport.boundAddress().publishAddress()).address().getPort()) - .protocol("https") - .method("GET") - .path("/_nodes"); - requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, - UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, - new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))); - HttpResponse response = requestBuilder.execute(); - assertThat(response.getStatusCode(), is(200)); + CloseableHttpClient httpClient = HttpClients.custom().setSSLContext(sc).build(); + try (RestClient restClient = createRestClient(httpClient, "https")) { + try (Response response = restClient.performRequest("GET", "/_nodes", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, + new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) { + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } } } } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutSSLTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutSSLTests.java index 2902b87d2fc..6cea5fa9b08 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutSSLTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiWithoutSSLTests.java @@ -5,19 +5,18 @@ */ package org.elasticsearch.xpack.security.authc.pki; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Response; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.http.HttpServerTransport; -import org.elasticsearch.xpack.security.authc.support.SecuredString; -import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; +import org.elasticsearch.xpack.security.authc.support.SecuredString; +import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; + +import java.util.Collections; import static org.hamcrest.Matchers.is; @@ -44,17 +43,11 @@ public class PkiWithoutSSLTests extends SecurityIntegTestCase { } public void testThatHttpWorks() throws Exception { - HttpServerTransport httpServerTransport = internalCluster().getDataNodeInstance(HttpServerTransport.class); - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient) - .httpTransport(httpServerTransport) - .method("GET") - .path("/_nodes"); - requestBuilder.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, - UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, - new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))); - HttpResponse response = requestBuilder.execute(); - assertThat(response.getStatusCode(), is(200)); + try (Response response = getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null, + new BasicHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, + new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) { + assertThat(response.getStatusLine().getStatusCode(), is(200)); } } } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java index 705dcbe9d5f..af998349d5a 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java @@ -64,6 +64,7 @@ import java.util.Collections; import java.util.IdentityHashMap; import java.util.Set; +import static java.util.Collections.emptyList; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; @@ -94,7 +95,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase { Collections.emptyMap(), Collections.emptyMap()); SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); mapperService = new MapperService(indexSettings, analysisService, similarityService, - new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry(), () -> null); + new IndicesModule(new NamedWriteableRegistry(), emptyList()).getMapperRegistry(), () -> null); ShardId shardId = new ShardId(index, 0); licenseState = mock(SecurityLicenseState.class); diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java index 52e94e51cae..ad979915726 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestAuthenticateActionTests.java @@ -5,6 +5,10 @@ */ package org.elasticsearch.xpack.security.rest.action; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.security.authc.support.SecuredString; @@ -12,10 +16,10 @@ import org.elasticsearch.xpack.security.authz.InternalAuthorizationService; import org.elasticsearch.xpack.security.user.AnonymousUser; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.elasticsearch.test.rest.json.JsonPath; import org.junit.BeforeClass; +import java.util.Collections; import java.util.List; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -47,32 +51,40 @@ public class RestAuthenticateActionTests extends SecurityIntegTestCase { } public void testAuthenticateApi() throws Exception { - HttpResponse response = httpClient().method("GET").path("/_xpack/security/_authenticate") - .addHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, - new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))) - .execute(); - - assertThat(response.getStatusCode(), is(200)); - JsonPath jsonPath = new JsonPath(response.getBody()); - assertThat(jsonPath.evaluate("username").toString(), equalTo(SecuritySettingsSource.DEFAULT_USER_NAME)); - List roles = (List) jsonPath.evaluate("roles"); - assertThat(roles.size(), is(1)); - assertThat(roles, contains(SecuritySettingsSource.DEFAULT_ROLE)); + try (Response response = getRestClient().performRequest( + "GET", "/_xpack/security/_authenticate", Collections.emptyMap(), null, + new BasicHeader("Authorization", basicAuthHeaderValue(SecuritySettingsSource.DEFAULT_USER_NAME, + new SecuredString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))))) { + assertThat(response.getStatusLine().getStatusCode(), is(200)); + JsonPath jsonPath = new JsonPath(EntityUtils.toString(response.getEntity())); + assertThat(jsonPath.evaluate("username").toString(), equalTo(SecuritySettingsSource.DEFAULT_USER_NAME)); + @SuppressWarnings("unchecked") + List roles = (List) jsonPath.evaluate("roles"); + assertThat(roles.size(), is(1)); + assertThat(roles, contains(SecuritySettingsSource.DEFAULT_ROLE)); + } } public void testAuthenticateApiWithoutAuthentication() throws Exception { - HttpResponse response = httpClient().method("GET").path("/_xpack/security/_authenticate") - .execute(); - - if (anonymousEnabled) { - assertThat(response.getStatusCode(), is(200)); - JsonPath jsonPath = new JsonPath(response.getBody()); - assertThat(jsonPath.evaluate("username").toString(), equalTo("anon")); - List roles = (List) jsonPath.evaluate("roles"); - assertThat(roles.size(), is(2)); - assertThat(roles, contains(SecuritySettingsSource.DEFAULT_ROLE, "foo")); - } else { - assertThat(response.getStatusCode(), is(401)); + try (Response response = getRestClient().performRequest("GET", "/_xpack/security/_authenticate", + Collections.emptyMap(), null)) { + if (anonymousEnabled) { + assertThat(response.getStatusLine().getStatusCode(), is(200)); + JsonPath jsonPath = new JsonPath(EntityUtils.toString(response.getEntity())); + assertThat(jsonPath.evaluate("username").toString(), equalTo("anon")); + @SuppressWarnings("unchecked") + List roles = (List) jsonPath.evaluate("roles"); + assertThat(roles.size(), is(2)); + assertThat(roles, contains(SecuritySettingsSource.DEFAULT_ROLE, "foo")); + } else { + fail("request should have failed"); + } + } catch(ResponseException e) { + if (anonymousEnabled) { + fail("request should have succeeded"); + } else { + assertThat(e.getResponse().getStatusLine().getStatusCode(), is(401)); + } } } } diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslClientAuthTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslClientAuthTests.java index ccdfa853aa0..db8836dbf19 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslClientAuthTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslClientAuthTests.java @@ -9,20 +9,21 @@ import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.ssl.ClientSSLService; import org.elasticsearch.xpack.security.ssl.SSLConfiguration.Global; import org.elasticsearch.xpack.security.transport.netty.SecurityNettyHttpServerTransport; import org.elasticsearch.xpack.security.transport.netty.SecurityNettyTransport; import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.elasticsearch.transport.Transport; import org.elasticsearch.xpack.XPackPlugin; @@ -30,10 +31,12 @@ import javax.net.ssl.SSLHandshakeException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.test.SecuritySettingsSource.getSSLSettingsForStore; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class SslClientAuthTests extends SecurityIntegTestCase { @Override @@ -59,14 +62,8 @@ public class SslClientAuthTests extends SecurityIntegTestCase { SSLContexts.createDefault(), SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); - CloseableHttpClient client = HttpClients.custom().setSSLSocketFactory(socketFactory).build(); - - try { - new HttpRequestBuilder(client) - .httpTransport(internalCluster().getInstance(HttpServerTransport.class)) - .method("GET").path("/") - .protocol("https") - .execute(); + try (RestClient restClient = createRestClient(HttpClients.custom().setSSLSocketFactory(socketFactory).build(), "https")) { + restClient.performRequest("GET", "/", Collections.emptyMap(), null); fail("Expected SSLHandshakeException"); } catch (SSLHandshakeException e) { assertThat(e.getMessage(), containsString("unable to find valid certification path to requested target")); @@ -85,13 +82,13 @@ public class SslClientAuthTests extends SecurityIntegTestCase { CloseableHttpClient client = HttpClients.custom().setSSLSocketFactory(socketFactory).build(); - HttpResponse response = new HttpRequestBuilder(client) - .httpTransport(internalCluster().getInstance(HttpServerTransport.class)) - .method("GET").path("/") - .protocol("https") - .addHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword())) - .execute(); - assertThat(response.getBody(), containsString("You Know, for Search")); + try (RestClient restClient = createRestClient(client, "https")) { + try (Response response = restClient.performRequest("GET", "/", Collections.emptyMap(), null, + new BasicHeader("Authorization", basicAuthHeaderValue(transportClientUsername(), transportClientPassword())))) { + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(EntityUtils.toString(response.getEntity()), containsString("You Know, for Search")); + } + } } public void testThatTransportWorksWithoutSslClientAuth() throws Exception { diff --git a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java index 2458033d328..ee9fe3a43c2 100644 --- a/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java @@ -5,25 +5,16 @@ */ package org.elasticsearch.xpack.security.user; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.elasticsearch.common.io.Streams; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.xpack.security.authz.InternalAuthorizationService; import org.elasticsearch.test.SecurityIntegTestCase; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.Locale; +import java.util.Collections; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -51,28 +42,22 @@ public class AnonymousUserIntegTests extends SecurityIntegTestCase { } public void testAnonymousViaHttp() throws Exception { - try (CloseableHttpClient client = HttpClients.createDefault(); - CloseableHttpResponse response = client.execute(new HttpGet(getNodeUrl() + "_nodes"))) { - int statusCode = response.getStatusLine().getStatusCode(); - String data = Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8)); + try { + getRestClient().performRequest("GET", "/_nodes", Collections.emptyMap(), null); + fail("request should have failed"); + } catch(ResponseException e) { + int statusCode = e.getResponse().getStatusLine().getStatusCode(); + Response response = e.getResponse(); if (authorizationExceptionsEnabled) { assertThat(statusCode, is(403)); - assertThat(response.getFirstHeader("WWW-Authenticate"), nullValue()); - assertThat(data, containsString("security_exception")); + assertThat(response.getHeader("WWW-Authenticate"), nullValue()); + assertThat(e.getResponseBody(), containsString("security_exception")); } else { assertThat(statusCode, is(401)); - assertThat(response.getFirstHeader("WWW-Authenticate"), notNullValue()); - assertThat(response.getFirstHeader("WWW-Authenticate").getValue(), containsString("Basic")); - assertThat(data, containsString("security_exception")); + assertThat(response.getHeader("WWW-Authenticate"), notNullValue()); + assertThat(response.getHeader("WWW-Authenticate"), containsString("Basic")); + assertThat(e.getResponseBody(), containsString("security_exception")); } } } - - private String getNodeUrl() { - TransportAddress transportAddress = - randomFrom(internalCluster().getInstance(HttpServerTransport.class).boundAddress().boundAddresses()); - assertThat(transportAddress, is(instanceOf(InetSocketTransportAddress.class))); - InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) transportAddress; - return String.format(Locale.ROOT, "http://%s:%s/", "localhost", inetSocketTransportAddress.address().getPort()); - } } diff --git a/elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/actions b/elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/actions index a96611f6b02..0f984306848 100644 --- a/elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/actions +++ b/elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/actions @@ -1,4 +1,3 @@ -cluster:admin/render/template/search cluster:admin/repository/delete cluster:admin/repository/get cluster:admin/repository/put diff --git a/elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/handlers b/elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/handlers index 7c9c51bfba0..0aa0be82698 100644 --- a/elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/handlers +++ b/elasticsearch/x-pack/security/src/test/resources/org/elasticsearch/transport/handlers @@ -1,4 +1,3 @@ -cluster:admin/render/template/search cluster:admin/snapshot/status[nodes] cluster:admin/snapshot/status[nodes][n] cluster:admin/tasks/cancel[n] diff --git a/elasticsearch/x-pack/security/src/test/resources/rest-api-spec/test/roles/10_basic.yaml b/elasticsearch/x-pack/security/src/test/resources/rest-api-spec/test/roles/10_basic.yaml index 6f4695a8c18..9e5898103c6 100644 --- a/elasticsearch/x-pack/security/src/test/resources/rest-api-spec/test/roles/10_basic.yaml +++ b/elasticsearch/x-pack/security/src/test/resources/rest-api-spec/test/roles/10_basic.yaml @@ -22,6 +22,21 @@ } - match: { role: { created: true } } + - do: + xpack.security.put_role: + name: "backwards_role" + body: > + { + "cluster": ["all"], + "indices": [ + { + "privileges": ["all"], + "names": "*" + } + ] + } + - match: { role: { created: true } } + - do: xpack.security.put_user: username: "joe" diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/http/HttpResponse.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/http/HttpResponse.java index 1c66dc2f1a5..ca469cba782 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/http/HttpResponse.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/common/http/HttpResponse.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.common.http; -import com.google.common.collect.ImmutableMap; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; @@ -87,11 +86,11 @@ public class HttpResponse implements ToXContent { * in the payload */ public Map> headers() { - ImmutableMap.Builder> builder = ImmutableMap.builder(); + MapBuilder> builder = MapBuilder.newMapBuilder(); for (Map.Entry entry : headers.entrySet()) { builder.put(entry.getKey().toLowerCase(Locale.ROOT), Arrays.asList(entry.getValue())); } - return builder.build(); + return builder.immutableMap(); } public String[] header(String header) { diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Attachment.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Attachment.java index f05a995f995..34b848bc55d 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Attachment.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Attachment.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.notification.email; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.inject.Provider; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -16,19 +17,26 @@ import javax.activation.DataHandler; import javax.activation.DataSource; import javax.activation.FileDataSource; import javax.mail.MessagingException; -import javax.mail.Part; import javax.mail.internet.MimeBodyPart; import javax.mail.util.ByteArrayDataSource; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Path; +import static javax.mail.Part.ATTACHMENT; +import static javax.mail.Part.INLINE; + /** * */ public abstract class Attachment extends BodyPartSource { - protected Attachment(String id, String name, String contentType) { + private final boolean inline; + + protected Attachment(String id, String name, String contentType, boolean inline) { super(id, name, contentType); + this.inline = inline; } @Override @@ -36,13 +44,17 @@ public abstract class Attachment extends BodyPartSource { MimeBodyPart part = new MimeBodyPart(); part.setContentID(id); part.setFileName(name); - part.setDisposition(Part.ATTACHMENT); + part.setDisposition(inline ? INLINE : ATTACHMENT); writeTo(part); return part; } public abstract String type(); + public boolean isInline() { + return inline; + } + /** * intentionally not emitting path as it may come as an information leak */ @@ -65,22 +77,22 @@ public abstract class Attachment extends BodyPartSource { private final Path path; private final DataSource dataSource; - public File(String id, Path path) { - this(id, path.getFileName().toString(), path); + public File(String id, Path path, boolean inline) { + this(id, path.getFileName().toString(), path, inline); } - public File(String id, Path path, String contentType) { - this(id, path.getFileName().toString(), path, contentType); + public File(String id, Path path, String contentType, boolean inline) { + this(id, path.getFileName().toString(), path, contentType, inline); } @SuppressForbidden(reason = "uses toFile") - public File(String id, String name, Path path) { - this(id, name, path, fileTypeMap.getContentType(path.toFile())); + public File(String id, String name, Path path, boolean inline) { + this(id, name, path, fileTypeMap.getContentType(path.toFile()), inline); } @SuppressForbidden(reason = "uses toFile") - public File(String id, String name, Path path, String contentType) { - super(id, name, contentType); + public File(String id, String name, Path path, String contentType, boolean inline) { + super(id, name, contentType, inline); this.path = path; this.dataSource = new FileDataSource(path.toFile()); } @@ -105,16 +117,16 @@ public abstract class Attachment extends BodyPartSource { private final byte[] bytes; - public Bytes(String id, byte[] bytes, String contentType) { - this(id, id, bytes, contentType); + public Bytes(String id, byte[] bytes, String contentType, boolean inline) { + this(id, id, bytes, contentType, inline); } - public Bytes(String id, String name, byte[] bytes) { - this(id, name, bytes, fileTypeMap.getContentType(name)); + public Bytes(String id, String name, byte[] bytes, boolean inline) { + this(id, name, bytes, fileTypeMap.getContentType(name), inline); } - public Bytes(String id, String name, byte[] bytes, String contentType) { - super(id, name, contentType); + public Bytes(String id, String name, byte[] bytes, String contentType, boolean inline) { + super(id, name, contentType, inline); this.bytes = bytes; } @@ -134,6 +146,68 @@ public abstract class Attachment extends BodyPartSource { } } + public static class Stream extends Attachment { + + static final String TYPE = "stream"; + + private final Provider source; + + public Stream(String id, String name, boolean inline, Provider source) { + this(id, name, fileTypeMap.getContentType(name), inline, source); + } + + public Stream(String id, String name, String contentType, boolean inline, Provider source) { + super(id, name, contentType, inline); + this.source = source; + } + + @Override + public String type() { + return TYPE; + } + + @Override + protected void writeTo(MimeBodyPart part) throws MessagingException { + DataSource ds = new StreamDataSource(name, contentType, source); + DataHandler dh = new DataHandler(ds); + part.setDataHandler(dh); + } + + static class StreamDataSource implements DataSource { + + private final String name; + private final String contentType; + private final Provider source; + + public StreamDataSource(String name, String contentType, Provider source) { + this.name = name; + this.contentType = contentType; + this.source = source; + } + + @Override + public InputStream getInputStream() throws IOException { + return source.get(); + } + + @Override + public OutputStream getOutputStream() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public String getName() { + return name; + } + } + + } + public static class XContent extends Bytes { protected XContent(String id, ToXContent content, XContentType type) { @@ -141,7 +215,7 @@ public abstract class Attachment extends BodyPartSource { } protected XContent(String id, String name, ToXContent content, XContentType type) { - super(id, name, bytes(name, content, type), mimeType(type)); + super(id, name, bytes(name, content, type), mimeType(type), false); } static String mimeType(XContentType type) { diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Email.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Email.java index 99e8f33cd7d..c44f0f979dd 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Email.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Email.java @@ -49,11 +49,10 @@ public class Email implements ToXContent { final String textBody; final String htmlBody; final Map attachments; - final Map inlines; public Email(String id, Address from, AddressList replyTo, Priority priority, DateTime sentDate, AddressList to, AddressList cc, AddressList bcc, String subject, String textBody, String htmlBody, - Map attachments, Map inlines) { + Map attachments) { this.id = id; this.from = from; @@ -67,7 +66,6 @@ public class Email implements ToXContent { this.textBody = textBody; this.htmlBody = htmlBody; this.attachments = attachments; - this.inlines = inlines; } public String id() { @@ -118,10 +116,6 @@ public class Email implements ToXContent { return attachments; } - public Map inlines() { - return inlines; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -249,7 +243,6 @@ public class Email implements ToXContent { private String textBody; private String htmlBody; private Map attachments = new HashMap<>(); - private Map inlines = new HashMap<>(); private Builder() { } @@ -267,7 +260,6 @@ public class Email implements ToXContent { textBody = email.textBody; htmlBody = email.htmlBody; attachments.putAll(email.attachments); - inlines.putAll(email.inlines); return this; } @@ -358,14 +350,6 @@ public class Email implements ToXContent { return this; } - public Builder inline(Inline inline) { - if (inlines == null) { - throw new IllegalStateException("Email has already been built!"); - } - inlines.put(inline.id(), inline); - return this; - } - /** * Build the email. Note that adding items to attachments or inlines * after this is called is incorrect. @@ -373,9 +357,8 @@ public class Email implements ToXContent { public Email build() { assert id != null : "email id should not be null (should be set to the watch id"; Email email = new Email(id, from, replyTo, priority, sentDate, to, cc, bcc, subject, textBody, htmlBody, - unmodifiableMap(attachments), unmodifiableMap(inlines)); + unmodifiableMap(attachments)); attachments = null; - inlines = null; return email; } diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Inline.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Inline.java deleted file mode 100644 index d8043da494a..00000000000 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Inline.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.notification.email; - -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.inject.Provider; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.notification.email.support.BodyPartSource; - -import javax.activation.DataHandler; -import javax.activation.DataSource; -import javax.activation.FileDataSource; -import javax.mail.MessagingException; -import javax.mail.Part; -import javax.mail.internet.MimeBodyPart; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; - -/** - * - */ -public abstract class Inline extends BodyPartSource { - - protected Inline(String id, String name, String contentType) { - super(id, name, contentType); - } - - public abstract String type(); - - @Override - public final MimeBodyPart bodyPart() throws MessagingException { - MimeBodyPart part = new MimeBodyPart(); - part.setDisposition(Part.INLINE); - part.setContentID(id); - part.setFileName(name); - writeTo(part); - return part; - } - - /** - * intentionally not emitting path as it may come as an information leak - */ - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject() - .field("type", type()) - .field("id", id) - .field("name", name) - .field("content_type", contentType) - .endObject(); - } - - protected abstract void writeTo(MimeBodyPart part) throws MessagingException; - - public static class File extends Inline { - - static final String TYPE = "file"; - - private final Path path; - private DataSource dataSource; - - public File(String id, Path path) { - this(id, path.getFileName().toString(), path); - } - - @SuppressForbidden(reason = "uses toFile") - public File(String id, String name, Path path) { - this(id, name, path, fileTypeMap.getContentType(path.toFile())); - } - - @SuppressForbidden(reason = "uses toFile") - public File(String id, String name, Path path, String contentType) { - super(id, name, contentType); - this.path = path; - this.dataSource = new FileDataSource(path.toFile()); - } - - public Path path() { - return path; - } - - public String type() { - return TYPE; - } - - @Override - public void writeTo(MimeBodyPart part) throws MessagingException { - part.setDataHandler(new DataHandler(dataSource, contentType)); - } - } - - public static class Stream extends Inline { - - static final String TYPE = "stream"; - - private final Provider source; - - public Stream(String id, String name, Provider source) { - this(id, name, fileTypeMap.getContentType(name), source); - } - - public Stream(String id, String name, String contentType, Provider source) { - super(id, name, contentType); - this.source = source; - } - - @Override - public String type() { - return TYPE; - } - - @Override - protected void writeTo(MimeBodyPart part) throws MessagingException { - DataSource ds = new StreamDataSource(name, contentType, source); - DataHandler dh = new DataHandler(ds); - part.setDataHandler(dh); - } - - static class StreamDataSource implements DataSource { - - private final String name; - private final String contentType; - private final Provider source; - - public StreamDataSource(String name, String contentType, Provider source) { - this.name = name; - this.contentType = contentType; - this.source = source; - } - - @Override - public InputStream getInputStream() throws IOException { - return source.get(); - } - - @Override - public OutputStream getOutputStream() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public String getContentType() { - return contentType; - } - - @Override - public String getName() { - return name; - } - } - } - - public static class Bytes extends Stream { - - public Bytes(String id, String name, String contentType, byte[] bytes) { - super(id, name, contentType, new BytesStreamProvider(bytes)); - } - - public Bytes(String id, String name, String contentType, BytesReference bytes) { - super(id, name, contentType, new BytesStreamProvider(bytes)); - } - - static class BytesStreamProvider implements Provider { - - private final BytesReference bytes; - - public BytesStreamProvider(byte[] bytes) { - this(new BytesArray(bytes)); - } - - public BytesStreamProvider(BytesReference bytes) { - this.bytes = bytes; - } - - @Override - public InputStream get() { - return new ByteArrayInputStream(bytes.array(), bytes.arrayOffset(), bytes.length()); - } - } - } -} diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Profile.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Profile.java index b39ee8cb1a7..9c31063bded 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Profile.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/Profile.java @@ -92,15 +92,13 @@ public enum Profile implements ToXContent { alternative.addBodyPart(html); } - if (!email.inlines.isEmpty()) { - for (Inline inline : email.inlines.values()) { - related.addBodyPart(inline.bodyPart()); - } - } - if (!email.attachments.isEmpty()) { for (Attachment attachment : email.attachments.values()) { - mixed.addBodyPart(attachment.bodyPart()); + if (attachment.isInline()) { + related.addBodyPart(attachment.bodyPart()); + } else { + mixed.addBodyPart(attachment.bodyPart()); + } } } diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/DataAttachment.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/DataAttachment.java index 8678daae382..3327380b7c2 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/DataAttachment.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/DataAttachment.java @@ -54,10 +54,16 @@ public class DataAttachment implements EmailAttachmentParser.EmailAttachment { return Objects.hash(id, dataAttachment); } + @Override public String id() { return id; } + @Override + public boolean inline() { + return false; + } + public static Builder builder(String id) { return new Builder(id); } diff --git a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParser.java b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParser.java index a8ab0b1be38..b60cf89e5e3 100644 --- a/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParser.java +++ b/elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParser.java @@ -30,6 +30,14 @@ public interface EmailAttachmentParser { public interface Fields { + ParseField INLINE = new ParseField("inline"); ParseField REQUEST = new ParseField("request"); ParseField CONTENT_TYPE = new ParseField("content_type"); } @@ -56,6 +57,7 @@ public class HttpEmailAttachementParser implements EmailAttachmentParser responseHeaders = (Map) responseMap.get("headers"); assertThat(responseHeaders, not(hasKey("es.index"))); - assertThat(responseHeaders, hasEntry("es_index", Lists.newArrayList("value"))); + assertThat(responseHeaders, hasEntry("es_index", Collections.singletonList("value"))); assertThat(responseHeaders, not(hasKey("es.index.2"))); - assertThat(responseHeaders, hasEntry("es_index_2", Lists.newArrayList("value"))); + assertThat(responseHeaders, hasEntry("es_index_2", Collections.singletonList("value"))); } } diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/EmailTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/EmailTests.java index f5ce6fb7dab..563bbe523da 100644 --- a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/EmailTests.java +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/EmailTests.java @@ -41,9 +41,8 @@ public class EmailTests extends ESTestCase { String textBody = randomFrom("Random Body", "", null); String htmlBody = randomFrom("
BODY
", "", null); Map attachments = null; - Map inlines = null; - Email email = new Email(id, from, replyTo, priority, sentDate, to, cc, bcc, subject, textBody, htmlBody, attachments, inlines); + Email email = new Email(id, from, replyTo, priority, sentDate, to, cc, bcc, subject, textBody, htmlBody, attachments); XContentBuilder builder = XContentFactory.jsonBuilder(); email.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ManualPublicSmtpServersTester.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ManualPublicSmtpServersTester.java index caa645927df..b648136513e 100644 --- a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ManualPublicSmtpServersTester.java +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ManualPublicSmtpServersTester.java @@ -6,14 +6,13 @@ package org.elasticsearch.xpack.notification.email; import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cli.Terminal; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.common.secret.SecretService; -import java.io.IOException; import java.util.Collections; import java.util.Locale; @@ -26,13 +25,13 @@ public class ManualPublicSmtpServersTester { public static void main(String[] args) throws Exception { test(Profile.GMAIL, Settings.builder() - .put("xpack.notification.email.service.account.gmail.smtp.auth", true) - .put("xpack.notification.email.service.account.gmail.smtp.starttls.enable", true) - .put("xpack.notification.email.service.account.gmail.smtp.host", "smtp.gmail.com") - .put("xpack.notification.email.service.account.gmail.smtp.port", 587) - .put("xpack.notification.email.service.account.gmail.smtp.user", terminal.readText("username: ")) - .put("xpack.notification.email.service.account.gmail.smtp.password", new String(terminal.readSecret("password: "))) - .put("xpack.notification.email.service.account.gmail.email_defaults.to", terminal.readText("to: ")) + .put("xpack.notification.email.account.gmail.smtp.auth", true) + .put("xpack.notification.email.account.gmail.smtp.starttls.enable", true) + .put("xpack.notification.email.account.gmail.smtp.host", "smtp.gmail.com") + .put("xpack.notification.email.account.gmail.smtp.port", 587) + .put("xpack.notification.email.account.gmail.smtp.user", terminal.readText("username: ")) + .put("xpack.notification.email.account.gmail.smtp.password", new String(terminal.readSecret("password: "))) + .put("xpack.notification.email.account.gmail.email_defaults.to", terminal.readText("to: ")) ); } } @@ -41,13 +40,13 @@ public class ManualPublicSmtpServersTester { public static void main(String[] args) throws Exception { test(Profile.STANDARD, Settings.builder() - .put("xpack.notification.email.service.account.outlook.smtp.auth", true) - .put("xpack.notification.email.service.account.outlook.smtp.starttls.enable", true) - .put("xpack.notification.email.service.account.outlook.smtp.host", "smtp-mail.outlook.com") - .put("xpack.notification.email.service.account.outlook.smtp.port", 587) - .put("xpack.notification.email.service.account.outlook.smtp.user", "elastic.user@outlook.com") - .put("xpack.notification.email.service.account.outlook.smtp.password", "fantastic42") - .put("xpack.notification.email.service.account.outlook.email_defaults.to", "elastic.user@outlook.com") + .put("xpack.notification.email.account.outlook.smtp.auth", true) + .put("xpack.notification.email.account.outlook.smtp.starttls.enable", true) + .put("xpack.notification.email.account.outlook.smtp.host", "smtp-mail.outlook.com") + .put("xpack.notification.email.account.outlook.smtp.port", 587) + .put("xpack.notification.email.account.outlook.smtp.user", "elastic.user@outlook.com") + .put("xpack.notification.email.account.outlook.smtp.password", "fantastic42") + .put("xpack.notification.email.account.outlook.email_defaults.to", "elastic.user@outlook.com") .put() ); } @@ -57,15 +56,15 @@ public class ManualPublicSmtpServersTester { public static void main(String[] args) throws Exception { test(Profile.STANDARD, Settings.builder() - .put("xpack.notification.email.service.account.yahoo.smtp.starttls.enable", true) - .put("xpack.notification.email.service.account.yahoo.smtp.auth", true) - .put("xpack.notification.email.service.account.yahoo.smtp.host", "smtp.mail.yahoo.com") - .put("xpack.notification.email.service.account.yahoo.smtp.port", 587) - .put("xpack.notification.email.service.account.yahoo.smtp.user", "elastic.user@yahoo.com") - .put("xpack.notification.email.service.account.yahoo.smtp.password", "fantastic42") + .put("xpack.notification.email.account.yahoo.smtp.starttls.enable", true) + .put("xpack.notification.email.account.yahoo.smtp.auth", true) + .put("xpack.notification.email.account.yahoo.smtp.host", "smtp.mail.yahoo.com") + .put("xpack.notification.email.account.yahoo.smtp.port", 587) + .put("xpack.notification.email.account.yahoo.smtp.user", "elastic.user@yahoo.com") + .put("xpack.notification.email.account.yahoo.smtp.password", "fantastic42") // note: from must be set to the same authenticated user account - .put("xpack.notification.email.service.account.yahoo.email_defaults.from", "elastic.user@yahoo.com") - .put("xpack.notification.email.service.account.yahoo.email_defaults.to", "elastic.user@yahoo.com") + .put("xpack.notification.email.account.yahoo.email_defaults.from", "elastic.user@yahoo.com") + .put("xpack.notification.email.account.yahoo.email_defaults.to", "elastic.user@yahoo.com") ); } } @@ -75,42 +74,43 @@ public class ManualPublicSmtpServersTester { public static void main(String[] args) throws Exception { test(Profile.STANDARD, Settings.builder() - .put("xpack.notification.email.service.account.ses.smtp.auth", true) - .put("xpack.notification.email.service.account.ses.smtp.starttls.enable", true) - .put("xpack.notification.email.service.account.ses.smtp.starttls.required", true) - .put("xpack.notification.email.service.account.ses.smtp.host", "email-smtp.us-east-1.amazonaws.com") - .put("xpack.notification.email.service.account.ses.smtp.port", 587) - .put("xpack.notification.email.service.account.ses.smtp.user", terminal.readText("user: ")) - .put("xpack.notification.email.service.account.ses.email_defaults.from", "dummy.user@elasticsearch.com") - .put("xpack.notification.email.service.account.ses.email_defaults.to", terminal.readText("to: ")) - .put("xpack.notification.email.service.account.ses.smtp.password", + .put("xpack.notification.email.account.ses.smtp.auth", true) + .put("xpack.notification.email.account.ses.smtp.starttls.enable", true) + .put("xpack.notification.email.account.ses.smtp.starttls.required", true) + .put("xpack.notification.email.account.ses.smtp.host", "email-smtp.us-east-1.amazonaws.com") + .put("xpack.notification.email.account.ses.smtp.port", 587) + .put("xpack.notification.email.account.ses.smtp.user", terminal.readText("user: ")) + .put("xpack.notification.email.account.ses.email_defaults.from", "dummy.user@elasticsearch.com") + .put("xpack.notification.email.account.ses.email_defaults.to", terminal.readText("to: ")) + .put("xpack.notification.email.account.ses.smtp.password", new String(terminal.readSecret("password: "))) ); } } - static void test(Profile profile, Settings.Builder builder) throws Exception { - InternalEmailService service = startEmailService(builder); + static void test(Profile profile, Settings.Builder settingsBuilder) throws Exception { + String path = "/org/elasticsearch/xpack/watcher/actions/email/service/logo.png"; + if (InternalEmailServiceTests.class.getResourceAsStream(path) == null) { + throw new ElasticsearchException("Could not find logo at path {}", path); + } + + InternalEmailService service = startEmailService(settingsBuilder); try { - ToXContent content = new ToXContent() { - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject() - .field("key1", "value1") - .field("key2", "value2") - .field("key3", "value3") - .endObject(); - } - }; + ToXContent content = (xContentBuilder, params) -> xContentBuilder.startObject() + .field("key1", "value1") + .field("key2", "value2") + .field("key3", "value3") + .endObject(); Email email = Email.builder() .id("_id") .subject("_subject") .textBody("_text_body") - .htmlBody("html body

") + .htmlBody("html body

") .attach(new Attachment.XContent.Yaml("test.yml", content)) - .inline(new Inline.Stream("logo", "logo.jpg", () -> InternalEmailServiceTests.class.getResourceAsStream("logo.png"))) + .attach(new Attachment.Stream("logo.png", "logo.png", true, + () -> InternalEmailServiceTests.class.getResourceAsStream(path))) .build(); EmailService.EmailSent sent = service.send(email, null, profile); diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ProfileTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ProfileTests.java new file mode 100644 index 00000000000..bda4490a98a --- /dev/null +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/ProfileTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.notification.email; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.common.secret.SecretService; + +import javax.mail.BodyPart; +import javax.mail.Part; +import javax.mail.Session; +import javax.mail.internet.MimeMessage; +import javax.mail.internet.MimeMultipart; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + +public class ProfileTests extends ESTestCase { + + public void testThatInlineAttachmentsAreCreated() throws Exception { + String path = "/org/elasticsearch/xpack/watcher/actions/email/service/logo.png"; + Attachment attachment = new Attachment.Stream("inline.png", "inline.png", true, + () -> InternalEmailServiceTests.class.getResourceAsStream(path)); + + Email email = Email.builder() + .id("foo") + .from("foo@example.org") + .to("bar@example.org") + .subject(randomAsciiOfLength(10)) + .attach(attachment) + .build(); + + Settings settings = Settings.builder() + .put("default_account", "foo") + .put("account.foo.smtp.host", "_host") + .build(); + + Accounts accounts = new Accounts(settings, SecretService.Insecure.INSTANCE, logger); + Session session = accounts.account("foo").getConfig().createSession(); + MimeMessage mimeMessage = Profile.STANDARD.toMimeMessage(email, session); + + Object content = ((MimeMultipart) mimeMessage.getContent()).getBodyPart(0).getContent(); + assertThat(content, instanceOf(MimeMultipart.class)); + MimeMultipart multipart = (MimeMultipart) content; + + assertThat(multipart.getCount(), is(2)); + boolean foundInlineAttachment = false; + BodyPart bodyPart = null; + for (int i = 0; i < multipart.getCount(); i++) { + bodyPart = multipart.getBodyPart(i); + if (Part.INLINE.equalsIgnoreCase(bodyPart.getDisposition())) { + foundInlineAttachment = true; + break; + } + } + + assertThat("Expected to find an inline attachment in mime message, but didnt", foundInlineAttachment, is(true)); + } +} \ No newline at end of file diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParsersTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParsersTests.java index a165c876bea..8a7c1b90593 100644 --- a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParsersTests.java +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/EmailAttachmentParsersTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.notification.email.attachment; -import com.google.common.base.Charsets; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ToXContent; @@ -13,13 +12,14 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; import org.elasticsearch.xpack.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.common.http.Scheme; -import org.elasticsearch.xpack.watcher.watch.Payload; import org.elasticsearch.xpack.notification.email.Attachment; +import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; +import org.elasticsearch.xpack.watcher.watch.Payload; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -93,7 +93,8 @@ public class EmailAttachmentParsersTests extends ESTestCase { attachments.add(new DataAttachment("my-name.json", org.elasticsearch.xpack.notification.email.DataAttachment.JSON)); HttpRequestTemplate requestTemplate = HttpRequestTemplate.builder("localhost", 80).scheme(Scheme.HTTP).path("/").build(); - HttpRequestAttachment httpRequestAttachment = new HttpRequestAttachment("other-id", requestTemplate, null); + boolean inline = randomBoolean(); + HttpRequestAttachment httpRequestAttachment = new HttpRequestAttachment("other-id", requestTemplate, inline, null); attachments.add(httpRequestAttachment); EmailAttachments emailAttachments = new EmailAttachments(attachments); @@ -105,6 +106,9 @@ public class EmailAttachmentParsersTests extends ESTestCase { assertThat(builder.string(), containsString("other-id")); assertThat(builder.string(), containsString("localhost")); assertThat(builder.string(), containsString("/")); + if (inline) { + assertThat(builder.string(), containsString("inline")); + } } public void testThatTwoAttachmentsWithTheSameIdThrowError() throws Exception { @@ -161,7 +165,8 @@ public class EmailAttachmentParsersTests extends ESTestCase { @Override public Attachment toAttachment(WatchExecutionContext ctx, Payload payload, TestEmailAttachment attachment) { - return new Attachment.Bytes(attachment.id(), attachment.getValue().getBytes(Charsets.UTF_8), "personalContentType"); + return new Attachment.Bytes(attachment.id(), attachment.getValue().getBytes(StandardCharsets.UTF_8), + "personalContentType", false); } } @@ -193,6 +198,11 @@ public class EmailAttachmentParsersTests extends ESTestCase { return id; } + @Override + public boolean inline() { + return false; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.startObject(id) diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/HttpEmailAttachementParserTests.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/HttpEmailAttachementParserTests.java index 7c643ac9f15..f9c9e4d285c 100644 --- a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/HttpEmailAttachementParserTests.java +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/notification/email/attachment/HttpEmailAttachementParserTests.java @@ -37,15 +37,13 @@ import static org.mockito.Mockito.when; public class HttpEmailAttachementParserTests extends ESTestCase { - private SecretService.Insecure secretService; - private HttpAuthRegistry authRegistry; private HttpRequestTemplate.Parser httpRequestTemplateParser; private HttpClient httpClient; @Before public void init() throws Exception { - secretService = SecretService.Insecure.INSTANCE; - authRegistry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(secretService))); + SecretService.Insecure secretService = SecretService.Insecure.INSTANCE; + HttpAuthRegistry authRegistry = new HttpAuthRegistry(singletonMap(BasicAuth.TYPE, new BasicAuthFactory(secretService))); httpRequestTemplateParser = new HttpRequestTemplate.Parser(authRegistry); httpClient = mock(HttpClient.class); @@ -77,9 +75,12 @@ public class HttpEmailAttachementParserTests extends ESTestCase { if (configureContentType) { builder.field("content_type", "application/foo"); } + boolean isInline = randomBoolean(); + if (isInline) { + builder.field("inline", true); + } builder.endObject().endObject().endObject(); XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes()); - logger.info("JSON: {}", builder.string()); EmailAttachments emailAttachments = emailAttachmentsParser.parse(parser); assertThat(emailAttachments.getAttachments(), hasSize(1)); @@ -89,6 +90,7 @@ public class HttpEmailAttachementParserTests extends ESTestCase { attachments.get(0).toXContent(toXcontentBuilder, ToXContent.EMPTY_PARAMS); toXcontentBuilder.endObject(); assertThat(toXcontentBuilder.string(), is(builder.string())); - } + assertThat(attachments.get(0).inline(), is(isInline)); + } } diff --git a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java index 4e54f239126..d366e80fd7e 100644 --- a/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java +++ b/elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java @@ -5,26 +5,10 @@ */ package org.elasticsearch.xpack.test.rest; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.List; -import java.util.Map; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.BasicHttpClientConnectionManager; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -33,15 +17,25 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.license.plugin.TestUtils; -import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.client.RestTestResponse; import org.elasticsearch.test.rest.parser.RestTestParseException; +import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; +import org.elasticsearch.xpack.security.authc.support.SecuredString; +import org.elasticsearch.xpack.security.authz.store.ReservedRolesStore; import org.junit.After; import org.junit.Before; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.hamcrest.Matchers.is; public abstract class XPackRestTestCase extends ESRestTestCase { @@ -56,46 +50,16 @@ public abstract class XPackRestTestCase extends ESRestTestCase { return ESRestTestCase.createParameters(0, 1); } - @Before - public void startWatcher() throws Exception { - try (CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", - null, - url.getHost(), - url.getPort(), - "/_xpack/watcher/_start", null, null)); - request.addHeader("Authorization", BASIC_AUTH_VALUE); - try (CloseableHttpResponse response = client.execute(request)) { - } - } - } - - @After - public void stopWatcher() throws Exception { - try(CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - URL url = getClusterUrls()[0]; - HttpPut request = new HttpPut(new URI("http", - null, - url.getHost(), - url.getPort(), - "/_xpack/watcher/_stop", null, null)); - request.addHeader("Authorization", BASIC_AUTH_VALUE); - try (CloseableHttpResponse response = client.execute(request)) { - } - } - } - @Before public void installLicense() throws Exception { final XContentBuilder builder = XContentFactory.jsonBuilder(); TestUtils.generateSignedLicense("trial", TimeValue.timeValueHours(2)).toXContent(builder, ToXContent.EMPTY_PARAMS); final BytesReference bytes = builder.bytes(); try (XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes)) { - final List> bodies = Collections.singletonList(Collections.singletonMap("license", + final List> bodies = singletonList(singletonMap("license", parser.map())); - getAdminExecutionContext().callApi("license.post", Collections.singletonMap("acknowledge", "true"), - bodies, Collections.singletonMap("Authorization", BASIC_AUTH_VALUE)); + getAdminExecutionContext().callApi("license.post", singletonMap("acknowledge", "true"), + bodies, singletonMap("Authorization", BASIC_AUTH_VALUE)); } } @@ -103,43 +67,21 @@ public abstract class XPackRestTestCase extends ESRestTestCase { public void clearUsersAndRoles() throws Exception { // we cannot delete the .security index from a rest test since we aren't the internal user, lets wipe the data // TODO remove this once the built-in SUPERUSER role is added that can delete the index and we use the built in admin user here - try (CloseableHttpClient client = HttpClients.createMinimal(new BasicHttpClientConnectionManager())) { - final URL url = getClusterUrls()[0]; - HttpGet getUsersRequest = new HttpGet(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/security/user", null, null)); - getUsersRequest.addHeader("Authorization", BASIC_AUTH_VALUE); - try (CloseableHttpResponse closeableHttpResponse = client.execute(getUsersRequest)) { - assertThat(closeableHttpResponse.getStatusLine().getStatusCode(), is(200)); - String response = Streams.copyToString( - new InputStreamReader(closeableHttpResponse.getEntity().getContent(), StandardCharsets.UTF_8)); - Map responseMap = XContentFactory.xContent(response).createParser(response).map(); - - // in the structure of this API, the users are the keyset - for (String user : responseMap.keySet()) { - HttpDelete delete = new HttpDelete(new URI("http", null, url.getHost(), url.getPort(), - "/_xpack/security/user/" + user, null, null)); - delete.addHeader("Authorization", BASIC_AUTH_VALUE); - try (CloseableHttpResponse deleteResponse = client.execute(delete)) { - } - } + RestTestResponse response = getAdminExecutionContext().callApi("xpack.security.get_user", emptyMap(), emptyList(), emptyMap()); + @SuppressWarnings("unchecked") + Map users = (Map) response.getBody(); + for (String user: users.keySet()) { + if (ReservedRealm.isReserved(user) == false) { + getAdminExecutionContext().callApi("xpack.security.delete_user", singletonMap("username", user), emptyList(), emptyMap()); } + } - HttpGet getRolesRequest = new HttpGet(new URI("http", null, url.getHost(), url.getPort(), "/_xpack/security/role", - null, null)); - getRolesRequest.addHeader("Authorization", BASIC_AUTH_VALUE); - try (CloseableHttpResponse closeableHttpResponse = client.execute(getRolesRequest)) { - assertThat(closeableHttpResponse.getStatusLine().getStatusCode(), is(200)); - String response = Streams.copyToString( - new InputStreamReader(closeableHttpResponse.getEntity().getContent(), StandardCharsets.UTF_8)); - Map responseMap = XContentFactory.xContent(response).createParser(response).map(); - - // in the structure of this API, the users are the keyset - for (String role : responseMap.keySet()) { - HttpDelete delete = new HttpDelete(new URI("http", null, url.getHost(), url.getPort(), - "/_xpack/security/role/" + role, null, null)); - delete.addHeader("Authorization", BASIC_AUTH_VALUE); - try (CloseableHttpResponse deleteResponse = client.execute(delete)) { - } - } + response = getAdminExecutionContext().callApi("xpack.security.get_role", emptyMap(), emptyList(), emptyMap()); + @SuppressWarnings("unchecked") + Map roles = (Map) response.getBody(); + for (String role: roles.keySet()) { + if (ReservedRolesStore.isReserved(role) == false) { + getAdminExecutionContext().callApi("xpack.security.delete_role", singletonMap("name", role), emptyList(), emptyMap()); } } } diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index fc8623e42fb..960a63e039c 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -31,7 +31,6 @@ import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor; import org.elasticsearch.xpack.watcher.history.HistoryModule; import org.elasticsearch.xpack.watcher.history.HistoryStore; import org.elasticsearch.xpack.watcher.input.InputModule; -import org.elasticsearch.xpack.watcher.input.chain.ChainInputFactory; import org.elasticsearch.xpack.watcher.rest.action.RestAckWatchAction; import org.elasticsearch.xpack.watcher.rest.action.RestActivateWatchAction; import org.elasticsearch.xpack.watcher.rest.action.RestDeleteWatchAction; @@ -266,5 +265,5 @@ public class Watcher { "[.watcher-history-YYYY.MM.dd] are allowed to be created", value); } - + } diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/index/ExecutableIndexAction.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/index/ExecutableIndexAction.java index 1b90a7d10b6..b5703c18ff1 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/index/ExecutableIndexAction.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/index/ExecutableIndexAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.xpack.watcher.actions.Action; import org.elasticsearch.xpack.watcher.actions.ExecutableAction; import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; @@ -24,6 +23,7 @@ import org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.watcher.watch.Payload; +import org.joda.time.DateTime; import java.io.IOException; import java.util.HashMap; @@ -63,19 +63,9 @@ public class ExecutableIndexAction extends ExecutableAction { } IndexRequest indexRequest = new IndexRequest(); - indexRequest.index(action.index); indexRequest.type(action.docType); - - if (action.executionTimeField != null && !TimestampFieldMapper.NAME.equals(action.executionTimeField)) { - if (!(data instanceof HashMap)) { - data = new HashMap<>(data); // ensuring mutability - } - data.put(action.executionTimeField, WatcherDateTimeUtils.formatDate(ctx.executionTime())); - } else { - indexRequest.timestamp(WatcherDateTimeUtils.formatDate(ctx.executionTime())); - } - + data = addTimestampToDocument(data, ctx.executionTime()); indexRequest.source(jsonBuilder().prettyPrint().map(data)); if (ctx.simulateAction(actionId)) { @@ -100,14 +90,7 @@ public class ExecutableIndexAction extends ExecutableAction { IndexRequest indexRequest = new IndexRequest(); indexRequest.index(action.index); indexRequest.type(action.docType); - if (action.executionTimeField != null && !TimestampFieldMapper.NAME.equals(action.executionTimeField)) { - if (!(doc instanceof HashMap)) { - doc = new HashMap<>(doc); // ensuring mutability - } - doc.put(action.executionTimeField, WatcherDateTimeUtils.formatDate(ctx.executionTime())); - } else { - indexRequest.timestamp(WatcherDateTimeUtils.formatDate(ctx.executionTime())); - } + doc = addTimestampToDocument(doc, ctx.executionTime()); indexRequest.source(jsonBuilder().prettyPrint().map(doc)); bulkRequest.add(indexRequest); } @@ -121,6 +104,16 @@ public class ExecutableIndexAction extends ExecutableAction { return new IndexAction.Result.Success(new XContentSource(jsonBuilder.bytes(), XContentType.JSON)); } + private Map addTimestampToDocument(Map data, DateTime executionTime) { + if (action.executionTimeField != null) { + if (!(data instanceof HashMap)) { + data = new HashMap<>(data); // ensuring mutability + } + data.put(action.executionTimeField, WatcherDateTimeUtils.formatDate(executionTime)); + } + return data; + } + static void indexResponseToXContent(XContentBuilder builder, IndexResponse response) throws IOException { builder.startObject() .field("created", response.isCreated()) diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/InputBuilders.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/InputBuilders.java index d99fd153b4f..bd7183046d2 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/InputBuilders.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/InputBuilders.java @@ -6,14 +6,14 @@ package org.elasticsearch.xpack.watcher.input; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.xpack.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.input.chain.ChainInput; import org.elasticsearch.xpack.watcher.input.http.HttpInput; import org.elasticsearch.xpack.watcher.input.none.NoneInput; import org.elasticsearch.xpack.watcher.input.search.SearchInput; import org.elasticsearch.xpack.watcher.input.simple.SimpleInput; -import org.elasticsearch.xpack.common.http.HttpRequestTemplate; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; import org.elasticsearch.xpack.watcher.watch.Payload; import java.util.HashMap; @@ -31,12 +31,12 @@ public final class InputBuilders { return NoneInput.builder(); } - public static SearchInput.Builder searchInput(SearchRequest request) { + public static SearchInput.Builder searchInput(WatcherSearchTemplateRequest request) { return SearchInput.builder(request); } - public static SearchInput.Builder searchInput(SearchRequestBuilder builder) { - return searchInput(builder.request()); + public static SearchInput.Builder searchInput(SearchRequest request) { + return searchInput(new WatcherSearchTemplateRequest(request)); } public static SimpleInput.Builder simpleInput() { diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java index b4983110b95..64335419329 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/ExecutableSearchInput.java @@ -11,16 +11,15 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; import org.elasticsearch.xpack.watcher.input.ExecutableInput; -import org.elasticsearch.xpack.watcher.support.WatcherUtils; import org.elasticsearch.xpack.watcher.support.XContentFilterKeysUtils; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; import org.elasticsearch.xpack.watcher.watch.Payload; import java.util.Map; @@ -35,11 +34,14 @@ public class ExecutableSearchInput extends ExecutableInput extractKeys; private final @Nullable TimeValue timeout; private final @Nullable DateTimeZone dynamicNameTimeZone; - public SearchInput(SearchRequest searchRequest, @Nullable Set extractKeys, + public SearchInput(WatcherSearchTemplateRequest request, @Nullable Set extractKeys, @Nullable TimeValue timeout, @Nullable DateTimeZone dynamicNameTimeZone) { - this.searchRequest = searchRequest; + this.request = request; this.extractKeys = extractKeys; this.timeout = timeout; this.dynamicNameTimeZone = dynamicNameTimeZone; @@ -63,7 +62,7 @@ public class SearchInput implements Input { SearchInput that = (SearchInput) o; - if (!SearchRequestEquivalence.INSTANCE.equivalent(searchRequest, this.searchRequest)) return false; + if (request != null ? !request.equals(that.request) : that.request != null) return false; if (extractKeys != null ? !extractKeys.equals(that.extractKeys) : that.extractKeys != null) return false; if (timeout != null ? !timeout.equals(that.timeout) : that.timeout != null) return false; return !(dynamicNameTimeZone != null ? !dynamicNameTimeZone.equals(that.dynamicNameTimeZone) : that.dynamicNameTimeZone != null); @@ -71,15 +70,15 @@ public class SearchInput implements Input { @Override public int hashCode() { - int result = searchRequest.hashCode(); + int result = request != null ? request.hashCode() : 0; result = 31 * result + (extractKeys != null ? extractKeys.hashCode() : 0); result = 31 * result + (timeout != null ? timeout.hashCode() : 0); result = 31 * result + (dynamicNameTimeZone != null ? dynamicNameTimeZone.hashCode() : 0); return result; } - public SearchRequest getSearchRequest() { - return searchRequest; + public WatcherSearchTemplateRequest getRequest() { + return request; } public Set getExtractKeys() { @@ -97,8 +96,9 @@ public class SearchInput implements Input { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(Field.REQUEST.getPreferredName()); - builder = WatcherUtils.writeSearchRequest(searchRequest, builder, params); + if (request != null) { + builder.field(Field.REQUEST.getPreferredName(), request); + } if (extractKeys != null) { builder.field(Field.EXTRACT.getPreferredName(), extractKeys); } @@ -115,7 +115,7 @@ public class SearchInput implements Input { public static SearchInput parse(String watchId, XContentParser parser, QueryParseContext context, AggregatorParsers aggParsers, Suggesters suggesters) throws IOException { - SearchRequest request = null; + WatcherSearchTemplateRequest request = null; Set extract = null; TimeValue timeout = null; DateTimeZone dynamicNameTimeZone = null; @@ -127,7 +127,7 @@ public class SearchInput implements Input { currentFieldName = parser.currentName(); } else if (ParseFieldMatcher.STRICT.match(currentFieldName, Field.REQUEST)) { try { - request = WatcherUtils.readSearchRequest(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context, + request = WatcherSearchTemplateRequest.fromXContent(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context, aggParsers, suggesters); } catch (ElasticsearchParseException srpe) { throw new ElasticsearchParseException("could not parse [{}] input for watch [{}]. failed to parse [{}]", srpe, TYPE, @@ -170,7 +170,7 @@ public class SearchInput implements Input { return new SearchInput(request, extract, timeout, dynamicNameTimeZone); } - public static Builder builder(SearchRequest request) { + public static Builder builder(WatcherSearchTemplateRequest request) { return new Builder(request); } @@ -198,20 +198,19 @@ public class SearchInput implements Input { return builder; } builder.startObject(type); - builder.field(Field.REQUEST.getPreferredName()); - WatcherUtils.writeSearchRequest(request, builder, params); + builder.field(Field.REQUEST.getPreferredName(), new WatcherSearchTemplateRequest(request)); return builder.endObject(); } } public static class Builder implements Input.Builder { - private final SearchRequest request; + private final WatcherSearchTemplateRequest request; private final Set extractKeys = new HashSet<>(); private TimeValue timeout; private DateTimeZone dynamicNameTimeZone; - private Builder(SearchRequest request) { + private Builder(WatcherSearchTemplateRequest request) { this.request = request; } diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java index bf327158a7a..db111eb4f23 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java @@ -5,8 +5,6 @@ */ package org.elasticsearch.xpack.watcher.input.search; -import java.io.IOException; - import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.Loggers; @@ -17,10 +15,14 @@ import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.suggest.Suggesters; +import org.elasticsearch.xpack.common.ScriptServiceProxy; import org.elasticsearch.xpack.security.InternalClient; import org.elasticsearch.xpack.watcher.input.InputFactory; import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; + +import java.io.IOException; /** * @@ -33,15 +35,16 @@ public class SearchInputFactory extends InputFactory { final byte[] template = TemplateUtils.loadTemplate("/" + config.getFileName()+ ".json", INDEX_TEMPLATE_VERSION, - Pattern.quote("${xpack.watcher.template.version}")).getBytes(Charsets.UTF_8); + Pattern.quote("${xpack.watcher.template.version}")).getBytes(StandardCharsets.UTF_8); PutIndexTemplateRequest request = new PutIndexTemplateRequest(config.getTemplateName()).source(template); request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherUtils.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherUtils.java index 5d895c0a45c..cbc790d0988 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherUtils.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherUtils.java @@ -5,54 +5,22 @@ */ package org.elasticsearch.xpack.watcher.support; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.formatDate; - -import java.io.IOException; -import java.lang.reflect.Array; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.Template; -import org.elasticsearch.search.aggregations.AggregatorParsers; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.suggest.Suggesters; -import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; -import org.elasticsearch.xpack.watcher.watch.Payload; import org.joda.time.DateTime; -/** - */ +import java.io.IOException; +import java.lang.reflect.Array; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.formatDate; + public final class WatcherUtils { - static final ParseField INDICES_FIELD = new ParseField("indices"); - static final ParseField TYPES_FIELD = new ParseField("types"); - static final ParseField BODY_FIELD = new ParseField("body"); - static final ParseField SEARCH_TYPE_FIELD = new ParseField("search_type"); - static final ParseField INDICES_OPTIONS_FIELD = new ParseField("indices_options"); - static final ParseField EXPAND_WILDCARDS_FIELD = new ParseField("expand_wildcards"); - static final ParseField IGNORE_UNAVAILABLE_FIELD = new ParseField("ignore_unavailable"); - static final ParseField ALLOW_NO_INDICES_FIELD = new ParseField("allow_no_indices"); - static final ParseField TEMPLATE_FIELD = new ParseField("template"); - - public final static IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.lenientExpandOpen(); private WatcherUtils() { } @@ -62,214 +30,6 @@ public final class WatcherUtils { return XContentHelper.convertToMap(builder.bytes(), false).v2(); } - public static SearchRequest createSearchRequestFromPrototype(SearchRequest requestPrototype, WatchExecutionContext ctx, - Payload payload) throws IOException { - SearchRequest request = new SearchRequest() - .indicesOptions(requestPrototype.indicesOptions()) - .searchType(requestPrototype.searchType()) - .indices(requestPrototype.indices()) - .types(requestPrototype.types()); - - // Due the inconsistency with templates in ES 1.x, we maintain our own template format. - // This template format we use now, will become the template structure in ES 2.0 - Map watcherContextParams = Variables.createCtxModel(ctx, payload); - if (requestPrototype.source() != null) { - // Here we convert a watch search request body into an inline search template, - // this way if any Watcher related context variables are used, they will get resolved, - // by ES search template support - XContentBuilder builder = jsonBuilder(); - requestPrototype.source().toXContent(builder, ToXContent.EMPTY_PARAMS); - Template template = new Template(builder.string(), ScriptType.INLINE, null, builder.contentType(), watcherContextParams); - request.template(template); - } else if (requestPrototype.template() != null) { - // Here we convert watcher template into a ES core templates. Due to the different format we use, we - // convert to the template format used in ES core - Template template = requestPrototype.template(); - if (template.getParams() != null) { - watcherContextParams.putAll(template.getParams()); - } - template = new Template(template.getScript(), template.getType(), template.getLang(), template.getContentType(), - watcherContextParams); - request.template(template); - // } - } - // falling back to an empty body - return request; - } - - - /** - * Reads a new search request instance for the specified parser. - */ - public static SearchRequest readSearchRequest(XContentParser parser, SearchType searchType, QueryParseContext context, - AggregatorParsers aggParsers, Suggesters suggesters) - throws IOException { - IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; - SearchRequest searchRequest = new SearchRequest(); - - XContentParser.Token token; - String currentFieldName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - if (ParseFieldMatcher.STRICT.match(currentFieldName, BODY_FIELD)) { - searchRequest.source(SearchSourceBuilder.fromXContent(context, aggParsers, suggesters)); - } - } else if (token == XContentParser.Token.START_ARRAY) { - if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_FIELD)) { - List indices = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - indices.add(parser.textOrNull()); - } else { - throw new ElasticsearchParseException("could not read search request. expected string values in [" + - currentFieldName + "] field, but instead found [" + token + "]"); - } - } - searchRequest.indices(indices.toArray(new String[indices.size()])); - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TYPES_FIELD)) { - List types = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_STRING) { - types.add(parser.textOrNull()); - } else { - throw new ElasticsearchParseException("could not read search request. expected string values in [" + - currentFieldName + "] field, but instead found [" + token + "]"); - } - } - searchRequest.types(types.toArray(new String[types.size()])); - } else { - throw new ElasticsearchParseException("could not read search request. unexpected array field [" + - currentFieldName + "]"); - } - } else if (token == XContentParser.Token.START_OBJECT) { - if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_OPTIONS_FIELD)) { - boolean expandOpen = DEFAULT_INDICES_OPTIONS.expandWildcardsOpen(); - boolean expandClosed = DEFAULT_INDICES_OPTIONS.expandWildcardsClosed(); - boolean allowNoIndices = DEFAULT_INDICES_OPTIONS.allowNoIndices(); - boolean ignoreUnavailable = DEFAULT_INDICES_OPTIONS.ignoreUnavailable(); - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue()) { - if (ParseFieldMatcher.STRICT.match(currentFieldName, EXPAND_WILDCARDS_FIELD)) { - switch (parser.text()) { - case "all": - expandOpen = true; - expandClosed = true; - break; - case "open": - expandOpen = true; - expandClosed = false; - break; - case "closed": - expandOpen = false; - expandClosed = true; - break; - case "none": - expandOpen = false; - expandClosed = false; - break; - default: - throw new ElasticsearchParseException("could not read search request. unknown value [" + - parser.text() + "] for [" + currentFieldName + "] field "); - } - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, IGNORE_UNAVAILABLE_FIELD)) { - ignoreUnavailable = parser.booleanValue(); - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, ALLOW_NO_INDICES_FIELD)) { - allowNoIndices = parser.booleanValue(); - } else { - throw new ElasticsearchParseException("could not read search request. unexpected index option [" + - currentFieldName + "]"); - } - } else { - throw new ElasticsearchParseException("could not read search request. unexpected object field [" + - currentFieldName + "]"); - } - } - indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandOpen, expandClosed, - DEFAULT_INDICES_OPTIONS); - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TEMPLATE_FIELD)) { - Template template = Template.parse(parser, ParseFieldMatcher.STRICT); - searchRequest.template(template); - } else { - throw new ElasticsearchParseException("could not read search request. unexpected object field [" + - currentFieldName + "]"); - } - } else if (token == XContentParser.Token.VALUE_STRING) { - if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_FIELD)) { - String indicesStr = parser.text(); - searchRequest.indices(Strings.delimitedListToStringArray(indicesStr, ",", " \t")); - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TYPES_FIELD)) { - String typesStr = parser.text(); - searchRequest.types(Strings.delimitedListToStringArray(typesStr, ",", " \t")); - } else if (ParseFieldMatcher.STRICT.match(currentFieldName, SEARCH_TYPE_FIELD)) { - searchType = SearchType.fromString(parser.text().toLowerCase(Locale.ROOT), ParseFieldMatcher.EMPTY); - } else { - throw new ElasticsearchParseException("could not read search request. unexpected string field [" + - currentFieldName + "]"); - } - } else { - throw new ElasticsearchParseException("could not read search request. unexpected token [" + token + "]"); - } - } - - if (searchRequest.indices() == null) { - searchRequest.indices(Strings.EMPTY_ARRAY); - } - searchRequest.searchType(searchType); - searchRequest.indicesOptions(indicesOptions); - return searchRequest; - } - - /** - * Writes the searchRequest to the specified builder. - */ - public static XContentBuilder writeSearchRequest(SearchRequest searchRequest, XContentBuilder builder, - ToXContent.Params params) throws IOException { - if (searchRequest == null) { - builder.nullValue(); - return builder; - } - - builder.startObject(); - if (searchRequest.searchType() != null) { - builder.field(SEARCH_TYPE_FIELD.getPreferredName(), searchRequest.searchType().toString().toLowerCase(Locale.ENGLISH)); - } - if (searchRequest.indices() != null) { - builder.array(INDICES_FIELD.getPreferredName(), searchRequest.indices()); - } - if (searchRequest.types() != null) { - builder.array(TYPES_FIELD.getPreferredName(), searchRequest.types()); - } - if (searchRequest.source() != null) { - builder.field(BODY_FIELD.getPreferredName(), searchRequest.source()); - } - if (searchRequest.template() != null) { - builder.field(TEMPLATE_FIELD.getPreferredName(), searchRequest.template()); - } - - if (searchRequest.indicesOptions() != DEFAULT_INDICES_OPTIONS) { - IndicesOptions options = searchRequest.indicesOptions(); - builder.startObject(INDICES_OPTIONS_FIELD.getPreferredName()); - String value; - if (options.expandWildcardsClosed() && options.expandWildcardsOpen()) { - value = "all"; - } else if (options.expandWildcardsOpen()) { - value = "open"; - } else if (options.expandWildcardsClosed()) { - value = "closed"; - } else { - value = "none"; - } - builder.field(EXPAND_WILDCARDS_FIELD.getPreferredName(), value); - builder.field(IGNORE_UNAVAILABLE_FIELD.getPreferredName(), options.ignoreUnavailable()); - builder.field(ALLOW_NO_INDICES_FIELD.getPreferredName(), options.allowNoIndices()); - builder.endObject(); - } - return builder.endObject(); - } - public static Map flattenModel(Map map) { Map result = new HashMap<>(); flattenModel("", map, result); diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java new file mode 100644 index 00000000000..adc101d94e3 --- /dev/null +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateRequest.java @@ -0,0 +1,254 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.watcher.support.search; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.suggest.Suggesters; +import org.elasticsearch.xpack.watcher.support.Script; +import org.elasticsearch.xpack.watcher.support.SearchRequestEquivalence; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +/** + * A {@link WatcherSearchTemplateRequest} contains the search request and the eventual template that will + * be rendered as a script by {@link WatcherSearchTemplateService} before being executed. + */ +public class WatcherSearchTemplateRequest implements ToXContent { + + private final SearchRequest request; + private final @Nullable Script template; + + public WatcherSearchTemplateRequest(SearchRequest searchRequest, @Nullable Script template) { + this.request = Objects.requireNonNull(searchRequest); + this.template = template; + } + + public WatcherSearchTemplateRequest(SearchRequest request) { + this(request, null); + } + + public SearchRequest getRequest() { + return request; + } + + public Script getTemplate() { + return template; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (request != null) { + if (request.searchType() != null) { + builder.field(SEARCH_TYPE_FIELD.getPreferredName(), request.searchType().toString().toLowerCase(Locale.ENGLISH)); + } + if (request.indices() != null) { + builder.array(INDICES_FIELD.getPreferredName(), request.indices()); + } + if (request.types() != null) { + builder.array(TYPES_FIELD.getPreferredName(), request.types()); + } + if (request.source() != null) { + builder.field(BODY_FIELD.getPreferredName(), request.source()); + } + if (request.indicesOptions() != DEFAULT_INDICES_OPTIONS) { + IndicesOptions options = request.indicesOptions(); + builder.startObject(INDICES_OPTIONS_FIELD.getPreferredName()); + String value; + if (options.expandWildcardsClosed() && options.expandWildcardsOpen()) { + value = "all"; + } else if (options.expandWildcardsOpen()) { + value = "open"; + } else if (options.expandWildcardsClosed()) { + value = "closed"; + } else { + value = "none"; + } + builder.field(EXPAND_WILDCARDS_FIELD.getPreferredName(), value); + builder.field(IGNORE_UNAVAILABLE_FIELD.getPreferredName(), options.ignoreUnavailable()); + builder.field(ALLOW_NO_INDICES_FIELD.getPreferredName(), options.allowNoIndices()); + builder.endObject(); + } + } + if (template != null) { + builder.field(TEMPLATE_FIELD.getPreferredName(), template); + } + return builder.endObject(); + } + + + /** + * Reads a new watcher search request instance for the specified parser. + */ + public static WatcherSearchTemplateRequest fromXContent(XContentParser parser, SearchType searchType, QueryParseContext context, + AggregatorParsers aggParsers, Suggesters suggesters) + throws IOException { + IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; + SearchRequest searchRequest = new SearchRequest(); + Script template = null; + + XContentParser.Token token; + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if (ParseFieldMatcher.STRICT.match(currentFieldName, BODY_FIELD)) { + searchRequest.source(SearchSourceBuilder.fromXContent(context, aggParsers, suggesters)); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_FIELD)) { + List indices = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + indices.add(parser.textOrNull()); + } else { + throw new ElasticsearchParseException("could not read search request. expected string values in [" + + currentFieldName + "] field, but instead found [" + token + "]"); + } + } + searchRequest.indices(indices.toArray(new String[indices.size()])); + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TYPES_FIELD)) { + List types = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + types.add(parser.textOrNull()); + } else { + throw new ElasticsearchParseException("could not read search request. expected string values in [" + + currentFieldName + "] field, but instead found [" + token + "]"); + } + } + searchRequest.types(types.toArray(new String[types.size()])); + } else { + throw new ElasticsearchParseException("could not read search request. unexpected array field [" + + currentFieldName + "]"); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_OPTIONS_FIELD)) { + boolean expandOpen = DEFAULT_INDICES_OPTIONS.expandWildcardsOpen(); + boolean expandClosed = DEFAULT_INDICES_OPTIONS.expandWildcardsClosed(); + boolean allowNoIndices = DEFAULT_INDICES_OPTIONS.allowNoIndices(); + boolean ignoreUnavailable = DEFAULT_INDICES_OPTIONS.ignoreUnavailable(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (ParseFieldMatcher.STRICT.match(currentFieldName, EXPAND_WILDCARDS_FIELD)) { + switch (parser.text()) { + case "all": + expandOpen = true; + expandClosed = true; + break; + case "open": + expandOpen = true; + expandClosed = false; + break; + case "closed": + expandOpen = false; + expandClosed = true; + break; + case "none": + expandOpen = false; + expandClosed = false; + break; + default: + throw new ElasticsearchParseException("could not read search request. unknown value [" + + parser.text() + "] for [" + currentFieldName + "] field "); + } + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, IGNORE_UNAVAILABLE_FIELD)) { + ignoreUnavailable = parser.booleanValue(); + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, ALLOW_NO_INDICES_FIELD)) { + allowNoIndices = parser.booleanValue(); + } else { + throw new ElasticsearchParseException("could not read search request. unexpected index option [" + + currentFieldName + "]"); + } + } else { + throw new ElasticsearchParseException("could not read search request. unexpected object field [" + + currentFieldName + "]"); + } + } + indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandOpen, expandClosed, + DEFAULT_INDICES_OPTIONS); + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TEMPLATE_FIELD)) { + template = Script.parse(parser); + } else { + throw new ElasticsearchParseException("could not read search request. unexpected object field [" + + currentFieldName + "]"); + } + } else if (token == XContentParser.Token.VALUE_STRING) { + if (ParseFieldMatcher.STRICT.match(currentFieldName, INDICES_FIELD)) { + String indicesStr = parser.text(); + searchRequest.indices(Strings.delimitedListToStringArray(indicesStr, ",", " \t")); + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, TYPES_FIELD)) { + String typesStr = parser.text(); + searchRequest.types(Strings.delimitedListToStringArray(typesStr, ",", " \t")); + } else if (ParseFieldMatcher.STRICT.match(currentFieldName, SEARCH_TYPE_FIELD)) { + searchType = SearchType.fromString(parser.text().toLowerCase(Locale.ROOT), ParseFieldMatcher.EMPTY); + } else { + throw new ElasticsearchParseException("could not read search request. unexpected string field [" + + currentFieldName + "]"); + } + } else { + throw new ElasticsearchParseException("could not read search request. unexpected token [" + token + "]"); + } + } + + if (searchRequest.indices() == null) { + searchRequest.indices(Strings.EMPTY_ARRAY); + } + searchRequest.searchType(searchType); + searchRequest.indicesOptions(indicesOptions); + return new WatcherSearchTemplateRequest(searchRequest, template); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + WatcherSearchTemplateRequest that = (WatcherSearchTemplateRequest) o; + + if (!SearchRequestEquivalence.INSTANCE.equivalent(request, that.request)) return false; + return template != null ? template.equals(that.template) : that.template == null; + + } + + @Override + public int hashCode() { + int result = request != null ? request.hashCode() : 0; + result = 31 * result + (template != null ? template.hashCode() : 0); + return result; + } + + static final ParseField INDICES_FIELD = new ParseField("indices"); + static final ParseField TYPES_FIELD = new ParseField("types"); + static final ParseField BODY_FIELD = new ParseField("body"); + static final ParseField SEARCH_TYPE_FIELD = new ParseField("search_type"); + static final ParseField INDICES_OPTIONS_FIELD = new ParseField("indices_options"); + static final ParseField EXPAND_WILDCARDS_FIELD = new ParseField("expand_wildcards"); + static final ParseField IGNORE_UNAVAILABLE_FIELD = new ParseField("ignore_unavailable"); + static final ParseField ALLOW_NO_INDICES_FIELD = new ParseField("allow_no_indices"); + static final ParseField TEMPLATE_FIELD = new ParseField("template"); + + public final static IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.lenientExpandOpen(); +} diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java new file mode 100644 index 00000000000..ed00ea90dbd --- /dev/null +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.watcher.support.search; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.suggest.Suggesters; +import org.elasticsearch.xpack.common.ScriptServiceProxy; +import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; +import org.elasticsearch.xpack.watcher.support.Script; +import org.elasticsearch.xpack.watcher.support.Variables; +import org.elasticsearch.xpack.watcher.watch.Payload; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * {@link WatcherSearchTemplateService} renders {@link WatcherSearchTemplateRequest} before their execution. + */ +public class WatcherSearchTemplateService extends AbstractComponent { + + private static final String DEFAULT_LANG = "mustache"; + + private final ScriptServiceProxy scriptService; + private final ParseFieldMatcher parseFieldMatcher; + private final IndicesQueriesRegistry queryRegistry; + private final AggregatorParsers aggsParsers; + private final Suggesters suggesters; + + @Inject + public WatcherSearchTemplateService(Settings settings, ScriptServiceProxy scriptServiceProxy, + IndicesQueriesRegistry queryRegistry, AggregatorParsers aggregatorParsers, Suggesters suggesters) { + super(settings); + this.scriptService = scriptServiceProxy; + this.queryRegistry = queryRegistry; + this.aggsParsers = aggregatorParsers; + this.suggesters = suggesters; + this.parseFieldMatcher = new ParseFieldMatcher(settings); + } + + public SearchRequest createSearchRequestFromPrototype(WatcherSearchTemplateRequest prototype, WatchExecutionContext ctx, + Payload payload) throws IOException { + + SearchRequest request = new SearchRequest() + .indicesOptions(prototype.getRequest().indicesOptions()) + .searchType(prototype.getRequest().searchType()) + .indices(prototype.getRequest().indices()) + .types(prototype.getRequest().types()); + + Script template = null; + + // Due the inconsistency with templates in ES 1.x, we maintain our own template format. + // This template format we use now, will become the template structure in ES 2.0 + Map watcherContextParams = Variables.createCtxModel(ctx, payload); + + // Here we convert a watch search request body into an inline search template, + // this way if any Watcher related context variables are used, they will get resolved. + if (prototype.getRequest().source() != null) { + try (XContentBuilder builder = jsonBuilder()) { + prototype.getRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS); + template = Script.inline(builder.string()).lang(DEFAULT_LANG).params(watcherContextParams).build(); + } + + } else if (prototype.getTemplate() != null) { + // Here we convert watcher template into a ES core templates. Due to the different format we use, we + // convert to the template format used in ES core + Script templatePrototype = prototype.getTemplate(); + if (templatePrototype.params() != null) { + watcherContextParams.putAll(templatePrototype.params()); + } + + Script.Builder builder; + if (templatePrototype.type() == ScriptService.ScriptType.INLINE) { + builder = Script.inline(templatePrototype.script()); + } else if (templatePrototype.type() == ScriptService.ScriptType.FILE) { + builder = Script.file(templatePrototype.script()); + } else if (templatePrototype.type() == ScriptService.ScriptType.STORED) { + builder = Script.indexed(templatePrototype.script()); + } else { + builder = Script.defaultType(templatePrototype.script()); + } + template = builder.lang(templatePrototype.lang()).params(watcherContextParams).build(); + } + + request.source(convert(template)); + return request; + } + + /** + * Converts a {@link Script} to a {@link org.elasticsearch.search.builder.SearchSourceBuilder} + */ + private SearchSourceBuilder convert(Script template) throws IOException { + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); + if (template == null) { + // falling back to an empty body + return sourceBuilder; + } + + BytesReference source = (BytesReference) scriptService.executable(scriptService.compile(template), template.params()).run(); + if (source != null && source.length() > 0) { + try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) { + sourceBuilder.parseXContent(new QueryParseContext(queryRegistry, parser, parseFieldMatcher), aggsParsers, suggesters); + } + } + return sourceBuilder; + } +} diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/TransformBuilders.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/TransformBuilders.java index 2892516919b..b39e3b9655d 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/TransformBuilders.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/TransformBuilders.java @@ -6,8 +6,8 @@ package org.elasticsearch.xpack.watcher.transform; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.xpack.watcher.support.Script; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; import org.elasticsearch.xpack.watcher.transform.chain.ChainTransform; import org.elasticsearch.xpack.watcher.transform.script.ScriptTransform; import org.elasticsearch.xpack.watcher.transform.search.SearchTransform; @@ -20,12 +20,12 @@ public final class TransformBuilders { private TransformBuilders() { } - public static SearchTransform.Builder searchTransform(SearchRequest request) { + public static SearchTransform.Builder searchTransform(WatcherSearchTemplateRequest request) { return SearchTransform.builder(request); } - public static SearchTransform.Builder searchTransform(SearchRequestBuilder request) { - return searchTransform(request.request()); + public static SearchTransform.Builder searchTransform(SearchRequest request) { + return searchTransform(new WatcherSearchTemplateRequest(request)); } public static ScriptTransform.Builder scriptTransform(String script) { diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/ExecutableSearchTransform.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/ExecutableSearchTransform.java index 3e658ad3bfb..31b8d650ab3 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/ExecutableSearchTransform.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/ExecutableSearchTransform.java @@ -12,8 +12,8 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; -import org.elasticsearch.xpack.watcher.support.WatcherUtils; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; import org.elasticsearch.xpack.watcher.transform.ExecutableTransform; import org.elasticsearch.xpack.watcher.watch.Payload; @@ -25,12 +25,14 @@ public class ExecutableSearchTransform extends ExecutableTransform { - private final SearchRequest request; + private final WatcherSearchTemplateRequest request; private TimeValue timeout; private DateTimeZone dynamicNameTimeZone; - public Builder(SearchRequest request) { + public Builder(WatcherSearchTemplateRequest request) { this.request = request; } diff --git a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java index 8f7cf903af2..28d9e0aa599 100644 --- a/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java +++ b/elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java @@ -18,7 +18,9 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.xpack.security.InternalClient; +import org.elasticsearch.xpack.common.ScriptServiceProxy; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; import org.elasticsearch.xpack.watcher.transform.TransformFactory; /** @@ -32,14 +34,15 @@ public class SearchTransformFactory extends TransformFactory= 5").build()), - logger, scriptService); - - WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); - assertFalse(condition.execute(ctx).met()); - - client().prepareIndex("my-index", "my-type").setTimestamp("2005-01-01T00:40").setSource("{}").get(); - refresh(); - - response = client().prepareSearch("my-index") - .addAggregation(AggregationBuilders.dateHistogram("rate") - .field("_timestamp").dateHistogramInterval(DateHistogramInterval.HOUR).order(Histogram.Order.COUNT_DESC)) - .get(); - - ctx = mockExecutionContext("_name", new Payload.XContent(response)); - assertThat(condition.execute(ctx).met(), is(true)); - } - - public void testExecuteAccessHits() throws Exception { - ExecutableScriptCondition condition = new ExecutableScriptCondition( - new ScriptCondition(Script.inline("ctx.payload.hits?.hits[0]?._score == 1.0").build()), logger, scriptService); - InternalSearchHit hit = new InternalSearchHit(0, "1", new Text("type"), null); - hit.score(1f); - hit.shard(new SearchShardTarget("a", new Index("a", "testUUID"), 0)); - - InternalSearchResponse internalSearchResponse = new InternalSearchResponse( - new InternalSearchHits(new InternalSearchHit[]{hit}, 1L, 1f), null, null, null, false, false); - SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 500L, new ShardSearchFailure[0]); - - WatchExecutionContext ctx = mockExecutionContext("_watch_name", new Payload.XContent(response)); - assertThat(condition.execute(ctx).met(), is(true)); - hit.score(2f); - when(ctx.payload()).thenReturn(new Payload.XContent(response)); - assertThat(condition.execute(ctx).met(), is(false)); - } -} diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchTests.java index 852f80cf7d7..1185e3def78 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.watcher.execution; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase; import org.elasticsearch.xpack.watcher.test.WatcherTestUtils; import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent; @@ -21,7 +22,8 @@ import static org.hamcrest.Matchers.equalTo; */ public class TriggeredWatchTests extends AbstractWatcherIntegrationTestCase { public void testParser() throws Exception { - Watch watch = WatcherTestUtils.createTestWatch("fired_test", watcherHttpClient(), noopEmailService(), logger); + Watch watch = WatcherTestUtils.createTestWatch("fired_test", watcherHttpClient(), noopEmailService(), + watcherSearchTemplateService(), logger); ScheduleTriggerEvent event = new ScheduleTriggerEvent(watch.id(), DateTime.now(DateTimeZone.UTC), DateTime.now(DateTimeZone.UTC)); Wid wid = new Wid("_record", randomLong(), DateTime.now(DateTimeZone.UTC)); TriggeredWatch triggeredWatch = new TriggeredWatch(wid, event); @@ -38,4 +40,8 @@ public class TriggeredWatchTests extends AbstractWatcherIntegrationTestCase { private TriggeredWatch.Parser triggeredWatchParser() { return internalCluster().getInstance(TriggeredWatch.Parser.class); } + + protected WatcherSearchTemplateService watcherSearchTemplateService() { + return internalCluster().getInstance(WatcherSearchTemplateService.class); + } } diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java index 97bf5762fc6..67dfd582455 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.watcher.rest.action; -import com.google.common.collect.Lists; import org.elasticsearch.client.Client; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -18,6 +17,8 @@ import org.elasticsearch.xpack.watcher.client.WatcherClient; import org.elasticsearch.xpack.watcher.transport.actions.execute.ExecuteWatchRequestBuilder; import org.elasticsearch.xpack.watcher.trigger.TriggerService; +import java.util.Arrays; + import static org.hamcrest.core.Is.is; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -32,9 +33,9 @@ public class RestExecuteWatchActionTests extends ESTestCase { public void testThatFlagsCanBeSpecifiedViaParameters() throws Exception { String randomId = randomAsciiOfLength(10); - for (String recordExecution : Lists.newArrayList("true", "false", null)) { - for (String ignoreCondition : Lists.newArrayList("true", "false", null)) { - for (String debugCondition : Lists.newArrayList("true", "false", null)) { + for (String recordExecution : Arrays.asList("true", "false", null)) { + for (String ignoreCondition : Arrays.asList("true", "false", null)) { + for (String debugCondition : Arrays.asList("true", "false", null)) { ExecuteWatchRequestBuilder builder = new ExecuteWatchRequestBuilder(client); when(watcherClient.prepareExecuteWatch()).thenReturn(builder); diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherUtilsTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherUtilsTests.java index 68eea0e90ae..74536aa1e18 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherUtilsTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherUtilsTests.java @@ -6,23 +6,6 @@ package org.elasticsearch.xpack.watcher.support; -import static java.util.Collections.singletonMap; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.formatDate; -import static org.elasticsearch.xpack.watcher.support.WatcherUtils.DEFAULT_INDICES_OPTIONS; -import static org.elasticsearch.xpack.watcher.support.WatcherUtils.flattenModel; -import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType; -import static org.hamcrest.Matchers.arrayContainingInAnyOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.is; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; - import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; @@ -40,15 +23,30 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryParser; import org.elasticsearch.indices.query.IndicesQueriesRegistry; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.Template; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput; import org.elasticsearch.xpack.support.clock.SystemClock; -import org.elasticsearch.xpack.common.text.TextTemplate; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; import org.joda.time.DateTime; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput.DEFAULT_SEARCH_TYPE; +import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.formatDate; +import static org.elasticsearch.xpack.watcher.support.WatcherUtils.flattenModel; +import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + /** * */ @@ -100,6 +98,7 @@ public class WatcherUtilsTests extends ESTestCase { public void testSerializeSearchRequest() throws Exception { String[] randomIndices = generateRandomStringArray(5, 5, false); SearchRequest expectedRequest = new SearchRequest(randomIndices); + Script expectedTemplate = null; if (randomBoolean()) { String[] randomTypes = generateRandomStringArray(2, 5, false); @@ -107,7 +106,7 @@ public class WatcherUtilsTests extends ESTestCase { } expectedRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), - randomBoolean(), WatcherUtils.DEFAULT_INDICES_OPTIONS)); + randomBoolean(), WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS)); expectedRequest.searchType(getRandomSupportedSearchType()); SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()).size(11); @@ -122,30 +121,29 @@ public class WatcherUtilsTests extends ESTestCase { } } String text = randomAsciiOfLengthBetween(1, 5); - Template template = randomFrom( - new Template(text, ScriptType.INLINE, null, null, params), - new Template(text, ScriptType.FILE, null, null, params), - new Template(text, ScriptType.STORED, null, null, params) - ); - expectedRequest.template(template); + expectedTemplate = randomFrom(Script.inline(text), Script.file(text), Script.indexed(text)).params(params).build(); } + WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(expectedRequest, expectedTemplate); + XContentBuilder builder = jsonBuilder(); - builder = WatcherUtils.writeSearchRequest(expectedRequest, builder, ToXContent.EMPTY_PARAMS); + request.toXContent(builder, ToXContent.EMPTY_PARAMS); XContentParser parser = XContentHelper.createParser(builder.bytes()); assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); IndicesQueriesRegistry registry = new IndicesQueriesRegistry(); QueryParser queryParser = MatchAllQueryBuilder::fromXContent; registry.register(queryParser, MatchAllQueryBuilder.QUERY_NAME_FIELD); QueryParseContext context = new QueryParseContext(registry, parser, ParseFieldMatcher.STRICT); - SearchRequest result = WatcherUtils.readSearchRequest(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context, null, null); + WatcherSearchTemplateRequest result = WatcherSearchTemplateRequest.fromXContent(parser, DEFAULT_SEARCH_TYPE, context, null, null); - assertThat(result.indices(), arrayContainingInAnyOrder(expectedRequest.indices())); - assertThat(result.types(), arrayContainingInAnyOrder(expectedRequest.types())); - assertThat(result.indicesOptions(), equalTo(expectedRequest.indicesOptions())); - assertThat(result.searchType(), equalTo(expectedRequest.searchType())); - assertThat(result.source(), equalTo(searchSourceBuilder)); - assertThat(result.template(), equalTo(expectedRequest.template())); + assertThat(result.getRequest(), is(notNullValue())); + assertThat(result.getRequest().indices(), arrayContainingInAnyOrder(expectedRequest.indices())); + assertThat(result.getRequest().types(), arrayContainingInAnyOrder(expectedRequest.types())); + assertThat(result.getRequest().indicesOptions(), equalTo(expectedRequest.indicesOptions())); + assertThat(result.getRequest().searchType(), equalTo(expectedRequest.searchType())); + assertThat(result.getRequest().source(), equalTo(searchSourceBuilder)); + + assertThat(result.getTemplate(), equalTo(expectedTemplate)); } public void testDeserializeSearchRequest() throws Exception { @@ -172,10 +170,10 @@ public class WatcherUtilsTests extends ESTestCase { } } - IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; + IndicesOptions indicesOptions = WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS; if (randomBoolean()) { indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), - randomBoolean(), WatcherUtils.DEFAULT_INDICES_OPTIONS); + randomBoolean(), WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS); builder.startObject("indices_options") .field("allow_no_indices", indicesOptions.allowNoIndices()) .field("expand_wildcards", indicesOptions.expandWildcardsClosed() && indicesOptions.expandWildcardsOpen() ? "all" : @@ -201,7 +199,7 @@ public class WatcherUtilsTests extends ESTestCase { source = searchSourceBuilder.buildAsBytes(XContentType.JSON); builder.rawField("body", source); } - Template templateSource = null; + Script template = null; if (randomBoolean()) { Map params = new HashMap<>(); if (randomBoolean()) { @@ -211,14 +209,8 @@ public class WatcherUtilsTests extends ESTestCase { } } String text = randomAsciiOfLengthBetween(1, 5); - TextTemplate template = randomFrom( - TextTemplate.inline(text).params(params).build(), - TextTemplate.file(text).params(params).build(), - TextTemplate.indexed(text).params(params).build() - ); + template = randomFrom(Script.inline(text), Script.file(text), Script.indexed(text)) .params(params).build(); builder.field("template", template); - templateSource = new Template(template.getTemplate(), template.getType(), null, template.getContentType(), - template.getParams()); } builder.endObject(); @@ -228,14 +220,15 @@ public class WatcherUtilsTests extends ESTestCase { QueryParser queryParser = MatchAllQueryBuilder::fromXContent; registry.register(queryParser, MatchAllQueryBuilder.QUERY_NAME_FIELD); QueryParseContext context = new QueryParseContext(registry, parser, ParseFieldMatcher.STRICT); - SearchRequest result = WatcherUtils.readSearchRequest(parser, ExecutableSearchInput.DEFAULT_SEARCH_TYPE, context, null, null); + WatcherSearchTemplateRequest result = WatcherSearchTemplateRequest.fromXContent(parser, DEFAULT_SEARCH_TYPE, context, null, null); - assertThat(result.indices(), arrayContainingInAnyOrder(indices)); - assertThat(result.types(), arrayContainingInAnyOrder(types)); - assertThat(result.indicesOptions(), equalTo(indicesOptions)); - assertThat(result.searchType(), equalTo(searchType)); - assertThat(result.source(), equalTo(searchSourceBuilder)); - assertThat(result.template(), equalTo(templateSource)); + assertThat(result.getRequest(), is(notNullValue())); + assertThat(result.getRequest().indices(), arrayContainingInAnyOrder(indices)); + assertThat(result.getRequest().types(), arrayContainingInAnyOrder(types)); + assertThat(result.getRequest().indicesOptions(), equalTo(indicesOptions)); + assertThat(result.getRequest().searchType(), equalTo(searchType)); + assertThat(result.getRequest().source(), equalTo(searchSourceBuilder)); + assertThat(result.getTemplate(), equalTo(template)); } } diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java index f11d07a3f4a..3cff684748e 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java @@ -31,6 +31,13 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.common.ScriptServiceProxy; +import org.elasticsearch.xpack.common.http.HttpClient; +import org.elasticsearch.xpack.common.http.HttpMethod; +import org.elasticsearch.xpack.common.http.HttpRequestTemplate; +import org.elasticsearch.xpack.common.secret.Secret; +import org.elasticsearch.xpack.common.text.TextTemplate; +import org.elasticsearch.xpack.common.text.TextTemplateEngine; import org.elasticsearch.xpack.notification.email.Authentication; import org.elasticsearch.xpack.notification.email.EmailService; import org.elasticsearch.xpack.notification.email.EmailTemplate; @@ -49,15 +56,9 @@ import org.elasticsearch.xpack.watcher.execution.Wid; import org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput; import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput; import org.elasticsearch.xpack.watcher.input.simple.SimpleInput; -import org.elasticsearch.xpack.common.ScriptServiceProxy; -import org.elasticsearch.xpack.watcher.support.WatcherUtils; -import org.elasticsearch.xpack.common.http.HttpClient; -import org.elasticsearch.xpack.common.http.HttpMethod; -import org.elasticsearch.xpack.common.http.HttpRequestTemplate; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; -import org.elasticsearch.xpack.common.secret.Secret; -import org.elasticsearch.xpack.common.text.TextTemplate; -import org.elasticsearch.xpack.common.text.TextTemplateEngine; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; import org.elasticsearch.xpack.watcher.support.xcontent.ObjectPath; import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.watcher.transform.search.ExecutableSearchTransform; @@ -76,7 +77,6 @@ import javax.mail.internet.AddressException; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -122,7 +122,7 @@ public final class WatcherTestUtils { public static SearchRequest newInputSearchRequest(String... indices) { SearchRequest request = new SearchRequest(); request.indices(indices); - request.indicesOptions(WatcherUtils.DEFAULT_INDICES_OPTIONS); + request.indicesOptions(WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS); request.searchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE); return request; } @@ -175,13 +175,14 @@ public final class WatcherTestUtils { public static Watch createTestWatch(String watchName, HttpClient httpClient, EmailService emailService, - ESLogger logger) throws AddressException { - return createTestWatch(watchName, WatcherClientProxy.of(ESIntegTestCase.client()), httpClient, emailService, logger); + WatcherSearchTemplateService searchTemplateService, ESLogger logger) throws AddressException { + WatcherClientProxy client = WatcherClientProxy.of(ESIntegTestCase.client()); + return createTestWatch(watchName, client, httpClient, emailService, searchTemplateService, logger); } public static Watch createTestWatch(String watchName, WatcherClientProxy client, HttpClient httpClient, EmailService emailService, - ESLogger logger) throws AddressException { + WatcherSearchTemplateService searchTemplateService, ESLogger logger) throws AddressException { SearchRequest conditionRequest = newInputSearchRequest("my-condition-index").source(searchSource().query(matchAllQuery())); SearchRequest transformRequest = newInputSearchRequest("my-payload-index").source(searchSource().query(matchAllQuery())); @@ -229,12 +230,15 @@ public final class WatcherTestUtils { Map statuses = new HashMap<>(); statuses.put("_webhook", new ActionStatus(now)); statuses.put("_email", new ActionStatus(now)); + + SearchTransform searchTransform = new SearchTransform(new WatcherSearchTemplateRequest(transformRequest), null, null); + return new Watch( watchName, new ScheduleTrigger(new CronSchedule("0/5 * * * * ? *")), new ExecutableSimpleInput(new SimpleInput(new Payload.Simple(inputData)), logger), new ExecutableAlwaysCondition(logger), - new ExecutableSearchTransform(new SearchTransform(transformRequest, null, null), logger, client, null), + new ExecutableSearchTransform(searchTransform, logger, client, searchTemplateService, null), new TimeValue(0), new ExecutableActions(actions), metadata, @@ -247,7 +251,7 @@ public final class WatcherTestUtils { .put("script.indexed", "true") .put("path.home", createTempDir()) .build(); - ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Arrays.asList(ScriptServiceProxy.INSTANCE)); + ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.singletonList(ScriptServiceProxy.INSTANCE)); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.emptyList()); diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java index b552439b30e..5a1cb2d465f 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java @@ -7,20 +7,16 @@ package org.elasticsearch.xpack.watcher.test.integration; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.Callback; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.script.ScriptService.ScriptType; -import org.elasticsearch.script.Template; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.support.clock.SystemClock; import org.elasticsearch.xpack.watcher.client.WatchSourceBuilder; import org.elasticsearch.xpack.watcher.client.WatcherClient; import org.elasticsearch.xpack.watcher.condition.compare.CompareCondition; -import org.elasticsearch.xpack.watcher.support.WatcherUtils; -import org.elasticsearch.xpack.support.clock.SystemClock; +import org.elasticsearch.xpack.watcher.support.Script; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase; import org.elasticsearch.xpack.watcher.transport.actions.delete.DeleteWatchResponse; @@ -151,8 +147,8 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase { watchSource.field("unknown_field", "x"); watchSource.startObject("schedule").field("cron", "0/5 * * * * ? *").endObject(); - watchSource.startObject("condition").startObject("script").field("script", "return true").field("request"); - WatcherUtils.writeSearchRequest(newInputSearchRequest(), watchSource, ToXContent.EMPTY_PARAMS); + watchSource.startObject("condition").startObject("script").field("script", "return true"); + watchSource.field("request", new WatcherSearchTemplateRequest(newInputSearchRequest())); watchSource.endObject().endObject(); watchSource.endObject(); @@ -252,22 +248,20 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase { public void testConditionSearchWithSource() throws Exception { SearchSourceBuilder searchSourceBuilder = searchSource().query(matchQuery("level", "a")); - testConditionSearch(newInputSearchRequest("events").source(searchSourceBuilder)); + testConditionSearch(newInputSearchRequest("events").source(searchSourceBuilder), null); } public void testConditionSearchWithIndexedTemplate() throws Exception { SearchSourceBuilder searchSourceBuilder = searchSource().query(matchQuery("level", "a")); - client().admin().cluster().preparePutStoredScript() + assertAcked(client().admin().cluster().preparePutStoredScript() .setScriptLang("mustache") .setId("my-template") .setSource(jsonBuilder().startObject().field("template").value(searchSourceBuilder).endObject().bytes()) - .get(); + .get()); - Template template = new Template("my-template", ScriptType.STORED, null, null, null); + Script template = Script.indexed("my-template").lang("mustache").build(); SearchRequest searchRequest = newInputSearchRequest("events"); - // TODO (2.0 upgrade): move back to BytesReference instead of coverting to a string - searchRequest.template(template); - testConditionSearch(searchRequest); + testConditionSearch(searchRequest, template); } public void testInputFiltering() throws Exception { @@ -298,12 +292,7 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase { // Check that the input result payload has been filtered refresh(); - SearchResponse searchResponse = searchWatchRecords(new Callback() { - @Override - public void handle(SearchRequestBuilder builder) { - builder.setQuery(matchQuery("watch_id", "_name1")); - } - }); + SearchResponse searchResponse = searchWatchRecords(builder -> builder.setQuery(matchQuery("watch_id", "_name1"))); assertHitCount(searchResponse, 1); XContentSource source = xContentSource(searchResponse.getHits().getAt(0).getSourceRef()); assertThat(source.getValue("result.input.payload.hits.total"), equalTo((Object) 1)); @@ -379,17 +368,17 @@ public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase { } } - private void testConditionSearch(SearchRequest request) throws Exception { + private void testConditionSearch(SearchRequest request, Script template) throws Exception { // reset, so we don't miss event docs when we filter over the _timestamp field. timeWarp().clock().setTime(SystemClock.INSTANCE.nowUTC()); String watchName = "_name"; - assertAcked(prepareCreate("events").addMapping("event", "_timestamp", "enabled=true", "level", "type=text")); + assertAcked(prepareCreate("events").addMapping("event", "level", "type=text")); watcherClient().preparePutWatch(watchName) .setSource(watchBuilder() .trigger(schedule(interval("5s"))) - .input(searchInput(request)) + .input(searchInput(new WatcherSearchTemplateRequest(request, template))) .condition(compareCondition("ctx.payload.hits.total", CompareCondition.Op.GTE, 3L))) .get(); diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatcherSettingsFilterTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatcherSettingsFilterTests.java index a9b913575ba..4b9ea2d045d 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatcherSettingsFilterTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatcherSettingsFilterTests.java @@ -5,32 +5,29 @@ */ package org.elasticsearch.xpack.watcher.test.integration; +import org.apache.http.Header; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.elasticsearch.client.Response; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.xpack.security.authc.support.SecuredString; -import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; -import org.elasticsearch.test.rest.client.http.HttpResponse; import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase; import org.junit.After; import java.io.IOException; +import java.util.Collections; import java.util.Map; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase.SecuritySettings.TEST_PASSWORD; -import static org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase.SecuritySettings.TEST_USERNAME; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.is; -/** - * - */ public class WatcherSettingsFilterTests extends AbstractWatcherIntegrationTestCase { private CloseableHttpClient httpClient = HttpClients.createDefault(); @@ -52,37 +49,26 @@ public class WatcherSettingsFilterTests extends AbstractWatcherIntegrationTestCa } public void testGetSettingsSmtpPassword() throws Exception { - String body = executeRequest("GET", "/_nodes/settings", null, null).getBody(); - Map response = JsonXContent.jsonXContent.createParser(body).map(); - Map nodes = (Map) response.get("nodes"); - for (Object node : nodes.values()) { - Map settings = (Map) ((Map) node).get("settings"); - assertThat(XContentMapValues.extractValue("xpack.notification.email.account._email.smtp.user", settings), - is((Object) "_user")); - assertThat(XContentMapValues.extractValue("xpack.notification.email.account._email.smtp.password", settings), - nullValue()); + Header[] headers; + if (securityEnabled()) { + headers = new Header[] { + new BasicHeader(BASIC_AUTH_HEADER, + basicAuthHeaderValue(MarvelIntegTestCase.SecuritySettings.TEST_USERNAME, + new SecuredString(MarvelIntegTestCase.SecuritySettings.TEST_PASSWORD.toCharArray())))}; + } else { + headers = new Header[0]; } - } - - protected HttpResponse executeRequest(String method, String path, String body, Map params) throws IOException { - HttpServerTransport httpServerTransport = getInstanceFromMaster(HttpServerTransport.class); - HttpRequestBuilder requestBuilder = new HttpRequestBuilder(httpClient) - .httpTransport(httpServerTransport) - .method(method) - .path(path); - - if (params != null) { - for (Map.Entry entry : params.entrySet()) { - requestBuilder.addParam(entry.getKey(), entry.getValue()); + try (Response response = getRestClient().performRequest("GET", "/_nodes/settings", + Collections.emptyMap(), null, headers)) { + Map responseMap = JsonXContent.jsonXContent.createParser(response.getEntity().getContent()).map(); + Map nodes = (Map) responseMap.get("nodes"); + for (Object node : nodes.values()) { + Map settings = (Map) ((Map) node).get("settings"); + assertThat(XContentMapValues.extractValue("xpack.notification.email.account._email.smtp.user", settings), + is((Object) "_user")); + assertThat(XContentMapValues.extractValue("xpack.notification.email.account._email.smtp.password", settings), + nullValue()); } } - if (body != null) { - requestBuilder.body(body); - } - if (securityEnabled()) { - requestBuilder.addHeader(BASIC_AUTH_HEADER, - basicAuthHeaderValue(TEST_USERNAME, new SecuredString(TEST_PASSWORD.toCharArray()))); - } - return requestBuilder.execute(); } } diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java index fa34ad8c6d6..ea4a09715d2 100644 --- a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java @@ -5,14 +5,6 @@ */ package org.elasticsearch.xpack.watcher.watch; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.ESLogger; @@ -25,12 +17,31 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryParser; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.common.ScriptServiceProxy; +import org.elasticsearch.xpack.common.http.HttpClient; +import org.elasticsearch.xpack.common.http.HttpMethod; +import org.elasticsearch.xpack.common.http.HttpRequestTemplate; +import org.elasticsearch.xpack.common.http.auth.HttpAuthRegistry; +import org.elasticsearch.xpack.common.http.auth.basic.BasicAuthFactory; +import org.elasticsearch.xpack.common.secret.SecretService; +import org.elasticsearch.xpack.common.text.TextTemplate; +import org.elasticsearch.xpack.common.text.TextTemplateEngine; +import org.elasticsearch.xpack.notification.email.DataAttachment; +import org.elasticsearch.xpack.notification.email.EmailService; +import org.elasticsearch.xpack.notification.email.EmailTemplate; +import org.elasticsearch.xpack.notification.email.HtmlSanitizer; +import org.elasticsearch.xpack.notification.email.Profile; +import org.elasticsearch.xpack.notification.email.attachment.EmailAttachments; +import org.elasticsearch.xpack.notification.email.attachment.EmailAttachmentsParser; +import org.elasticsearch.xpack.support.clock.Clock; +import org.elasticsearch.xpack.support.clock.ClockMock; +import org.elasticsearch.xpack.support.clock.SystemClock; +import org.elasticsearch.xpack.watcher.WatcherLicensee; import org.elasticsearch.xpack.watcher.actions.ActionFactory; import org.elasticsearch.xpack.watcher.actions.ActionRegistry; import org.elasticsearch.xpack.watcher.actions.ActionStatus; import org.elasticsearch.xpack.watcher.actions.ActionWrapper; import org.elasticsearch.xpack.watcher.actions.ExecutableActions; -import org.elasticsearch.xpack.notification.email.DataAttachment; import org.elasticsearch.xpack.watcher.actions.email.EmailAction; import org.elasticsearch.xpack.watcher.actions.email.EmailActionFactory; import org.elasticsearch.xpack.watcher.actions.email.ExecutableEmailAction; @@ -68,28 +79,15 @@ import org.elasticsearch.xpack.watcher.input.search.SearchInputFactory; import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput; import org.elasticsearch.xpack.watcher.input.simple.SimpleInput; import org.elasticsearch.xpack.watcher.input.simple.SimpleInputFactory; -import org.elasticsearch.xpack.watcher.WatcherLicensee; import org.elasticsearch.xpack.watcher.support.Script; -import org.elasticsearch.xpack.watcher.support.WatcherUtils; -import org.elasticsearch.xpack.support.clock.Clock; -import org.elasticsearch.xpack.support.clock.ClockMock; -import org.elasticsearch.xpack.support.clock.SystemClock; -import org.elasticsearch.xpack.common.http.HttpClient; -import org.elasticsearch.xpack.common.http.HttpMethod; -import org.elasticsearch.xpack.common.http.HttpRequestTemplate; -import org.elasticsearch.xpack.common.http.auth.HttpAuthRegistry; -import org.elasticsearch.xpack.common.http.auth.basic.BasicAuthFactory; -import org.elasticsearch.xpack.common.ScriptServiceProxy; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; -import org.elasticsearch.xpack.common.secret.SecretService; -import org.elasticsearch.xpack.common.text.TextTemplate; -import org.elasticsearch.xpack.common.text.TextTemplateEngine; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; import org.elasticsearch.xpack.watcher.test.WatcherTestUtils; import org.elasticsearch.xpack.watcher.transform.ExecutableTransform; import org.elasticsearch.xpack.watcher.transform.TransformFactory; import org.elasticsearch.xpack.watcher.transform.TransformRegistry; import org.elasticsearch.xpack.watcher.transform.chain.ChainTransform; -import org.elasticsearch.xpack.watcher.transform.chain.ChainTransformFactory; import org.elasticsearch.xpack.watcher.transform.chain.ExecutableChainTransform; import org.elasticsearch.xpack.watcher.transform.script.ExecutableScriptTransform; import org.elasticsearch.xpack.watcher.transform.script.ScriptTransform; @@ -116,21 +114,23 @@ import org.elasticsearch.xpack.watcher.trigger.schedule.support.Month; import org.elasticsearch.xpack.watcher.trigger.schedule.support.MonthTimes; import org.elasticsearch.xpack.watcher.trigger.schedule.support.WeekTimes; import org.elasticsearch.xpack.watcher.trigger.schedule.support.YearTimes; -import org.elasticsearch.xpack.notification.email.EmailService; -import org.elasticsearch.xpack.notification.email.EmailTemplate; -import org.elasticsearch.xpack.notification.email.HtmlSanitizer; -import org.elasticsearch.xpack.notification.email.Profile; -import org.elasticsearch.xpack.notification.email.attachment.EmailAttachments; -import org.elasticsearch.xpack.notification.email.attachment.EmailAttachmentsParser; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.junit.Before; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; import static java.util.Collections.unmodifiableMap; - import static org.elasticsearch.xpack.watcher.input.InputBuilders.searchInput; +import static org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS; import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.matchAllRequest; import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule; import static org.hamcrest.Matchers.equalTo; @@ -153,6 +153,7 @@ public class WatchTests extends ESTestCase { private WatcherLicensee watcherLicensee; private ESLogger logger; private Settings settings = Settings.EMPTY; + private WatcherSearchTemplateService searchTemplateService; @Before public void init() throws Exception { @@ -166,6 +167,7 @@ public class WatchTests extends ESTestCase { watcherLicensee = mock(WatcherLicensee.class); authRegistry = new HttpAuthRegistry(singletonMap("basic", new BasicAuthFactory(secretService))); logger = Loggers.getLogger(WatchTests.class); + searchTemplateService = mock(WatcherSearchTemplateService.class); } public void testParserSelfGenerated() throws Exception { @@ -341,7 +343,7 @@ public class WatchTests extends ESTestCase { switch (type) { case SearchInput.TYPE: SearchInput searchInput = searchInput(WatcherTestUtils.newInputSearchRequest("idx")).build(); - return new ExecutableSearchInput(searchInput, logger, client, null); + return new ExecutableSearchInput(searchInput, logger, client, searchTemplateService, null); default: SimpleInput simpleInput = InputBuilders.simpleInput(singletonMap("_key", "_val")).build(); return new ExecutableSimpleInput(simpleInput, logger); @@ -355,7 +357,7 @@ public class WatchTests extends ESTestCase { IndicesQueriesRegistry queryRegistry = new IndicesQueriesRegistry(); QueryParser queryParser = MatchAllQueryBuilder::fromXContent; queryRegistry.register(queryParser, MatchAllQueryBuilder.QUERY_NAME_FIELD); - parsers.put(SearchInput.TYPE, new SearchInputFactory(settings, client, queryRegistry, null, null)); + parsers.put(SearchInput.TYPE, new SearchInputFactory(settings, client, queryRegistry, null, null, scriptService)); return new InputRegistry(Settings.EMPTY, parsers); default: parsers.put(SimpleInput.TYPE, new SimpleInputFactory(settings)); @@ -406,15 +408,19 @@ public class WatchTests extends ESTestCase { case ScriptTransform.TYPE: return new ExecutableScriptTransform(new ScriptTransform(Script.inline("_script").build()), logger, scriptService); case SearchTransform.TYPE: - return new ExecutableSearchTransform(new SearchTransform( - matchAllRequest(WatcherUtils.DEFAULT_INDICES_OPTIONS), timeout, timeZone), logger, client, null); + SearchTransform transform = new SearchTransform( + new WatcherSearchTemplateRequest(matchAllRequest(DEFAULT_INDICES_OPTIONS), null), timeout, timeZone); + return new ExecutableSearchTransform(transform, logger, client, searchTemplateService, null); default: // chain - ChainTransform chainTransform = new ChainTransform(Arrays.asList( - new SearchTransform(matchAllRequest(WatcherUtils.DEFAULT_INDICES_OPTIONS), timeout, timeZone), - new ScriptTransform(Script.inline("_script").build()))); + SearchTransform searchTransform = new SearchTransform( + new WatcherSearchTemplateRequest(matchAllRequest(DEFAULT_INDICES_OPTIONS), null), timeout, timeZone); + ScriptTransform scriptTransform = new ScriptTransform(Script.inline("_script").build()); + + ChainTransform chainTransform = new ChainTransform(Arrays.asList(searchTransform, scriptTransform)); return new ExecutableChainTransform(chainTransform, logger, Arrays.asList( new ExecutableSearchTransform(new SearchTransform( - matchAllRequest(WatcherUtils.DEFAULT_INDICES_OPTIONS), timeout, timeZone), logger, client, null), + new WatcherSearchTemplateRequest(matchAllRequest(DEFAULT_INDICES_OPTIONS), null), timeout, timeZone), + logger, client, searchTemplateService, null), new ExecutableScriptTransform(new ScriptTransform(Script.inline("_script").build()), logger, scriptService))); } } @@ -425,7 +431,7 @@ public class WatchTests extends ESTestCase { queryRegistry.register(queryParser, MatchAllQueryBuilder.QUERY_NAME_FIELD); Map factories = new HashMap<>(); factories.put(ScriptTransform.TYPE, new ScriptTransformFactory(settings, scriptService)); - factories.put(SearchTransform.TYPE, new SearchTransformFactory(settings, client, queryRegistry, null, null)); + factories.put(SearchTransform.TYPE, new SearchTransformFactory(settings, client, queryRegistry, null, null, scriptService)); TransformRegistry registry = new TransformRegistry(Settings.EMPTY, unmodifiableMap(factories)); return registry; } diff --git a/elasticsearch/x-pack/watcher/src/test/resources/rest-api-spec/api/xpack.watcher.ack_watch.json b/elasticsearch/x-pack/watcher/src/test/resources/rest-api-spec/api/xpack.watcher.ack_watch.json index 0193a94325d..23bc912a569 100644 --- a/elasticsearch/x-pack/watcher/src/test/resources/rest-api-spec/api/xpack.watcher.ack_watch.json +++ b/elasticsearch/x-pack/watcher/src/test/resources/rest-api-spec/api/xpack.watcher.ack_watch.json @@ -4,7 +4,7 @@ "methods": [ "PUT", "POST" ], "url": { "path": "/_xpack/watcher/watch/{watch_id}/_ack", - "paths": [ "/_xpack/watcher/watch/{watch_id}/_ack", "/_xpack/watcher/watch/{watch_id}/{action_id}/_ack"], + "paths": [ "/_xpack/watcher/watch/{watch_id}/_ack", "/_xpack/watcher/watch/{watch_id}/_ack/{action_id}"], "parts": { "watch_id": { "type" : "string",