From 62353ff8bc82fe6721cf8d82a50a0b2fa3931a2f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 28 Jul 2016 17:40:48 +0200 Subject: [PATCH] test: removed messy xpack mustache test module The tests have been moved back to xpack, turned into a rest test or removed. For example testing specific inline, file or stored mustache template functionality is already covered in the `lang-mustache` module. The smoke-test-watcher-with-mustache should tests watcher mustach specific things like the if the watcher variables are available. Original commit: elastic/x-pack-elasticsearch@e434bcd3fa7e1df84bda0e0bf323773e0e9a44c3 --- .../build.gradle | 11 - .../messy/tests/SearchInputIT.java | 393 ------------------ .../messy/tests/package-info.java | 23 - .../scripts/test_disk_template.mustache | 26 -- .../scripts/test_disk_template.mustache | 26 -- .../30_search_input_and_transform.yaml | 157 +++++++ .../SecurityCachePermissionTests.java} | 59 +-- .../test/integration/SearchInputTests.java | 199 +++++++++ .../integration/SearchTransformTests.java} | 262 +----------- 9 files changed, 371 insertions(+), 785 deletions(-) delete mode 100644 elasticsearch/qa/messy-test-xpack-with-mustache/build.gradle delete mode 100644 elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java delete mode 100644 elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java delete mode 100644 elasticsearch/qa/messy-test-xpack-with-mustache/src/test/resources/org/elasticsearch/xpack/watcher/input/search/config/scripts/test_disk_template.mustache delete mode 100644 elasticsearch/qa/messy-test-xpack-with-mustache/src/test/resources/org/elasticsearch/xpack/watcher/transform/search/config/scripts/test_disk_template.mustache create mode 100644 elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/30_search_input_and_transform.yaml rename elasticsearch/{qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java => x-pack/security/src/test/java/org/elasticsearch/integration/SecurityCachePermissionTests.java} (53%) create mode 100644 elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java rename elasticsearch/{qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java => x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java} (53%) diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/build.gradle b/elasticsearch/qa/messy-test-xpack-with-mustache/build.gradle deleted file mode 100644 index f2a8648e82e..00000000000 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/build.gradle +++ /dev/null @@ -1,11 +0,0 @@ - -/* - * Messy tests that depend on mustache directly. Fix these! - */ - -apply plugin: 'elasticsearch.messy-test' - -dependencies { - testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'testArtifacts') - testCompile project(path: ':modules:lang-mustache', configuration: 'runtime') -} diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java deleted file mode 100644 index dcb2e95263e..00000000000 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchInputIT.java +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.messy.tests; - -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.mustache.MustachePlugin; -import org.elasticsearch.search.aggregations.AggregatorParsers; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.suggest.Suggesters; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.xpack.common.text.TextTemplate; -import org.elasticsearch.xpack.watcher.actions.ActionWrapper; -import org.elasticsearch.xpack.watcher.actions.ExecutableActions; -import org.elasticsearch.xpack.watcher.condition.always.ExecutableAlwaysCondition; -import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext; -import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; -import org.elasticsearch.xpack.watcher.input.Input; -import org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput; -import org.elasticsearch.xpack.watcher.input.search.SearchInput; -import org.elasticsearch.xpack.watcher.input.search.SearchInputFactory; -import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput; -import org.elasticsearch.xpack.watcher.input.simple.SimpleInput; -import org.elasticsearch.xpack.watcher.support.WatcherScript; -import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; -import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; -import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; -import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource; -import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; -import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger; -import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent; -import org.elasticsearch.xpack.watcher.watch.Payload; -import org.elasticsearch.xpack.watcher.watch.Watch; -import org.elasticsearch.xpack.watcher.watch.WatchStatus; -import org.joda.time.DateTime; -import org.joda.time.chrono.ISOChronology; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import static java.util.Collections.emptyMap; -import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; -import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; -import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.joda.time.DateTimeZone.UTC; - -/** - */ -@ClusterScope(scope = SUITE, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, supportsDedicatedMasters = false, - numDataNodes = 1) -public class SearchInputIT extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - Collection> types = new ArrayList<>(); - types.addAll(super.nodePlugins()); - types.add(MustachePlugin.class); - types.add(CustomScriptContextPlugin.class); - return types; - } - - private static final String TEMPLATE_QUERY = "{\"query\":{\"bool\":{\"must\":{\"match\":{\"event_type\":{\"query\":\"a\"," + - "\"type\":\"boolean\"}}},\"filter\":{\"range\":{\"_timestamp\":" + - "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," + - "\"include_lower\":true,\"include_upper\":true}}}}}}"; - - @Override - public Settings nodeSettings(int nodeOrdinal) { - final Path tempDir = createTempDir(); - final Path configPath = tempDir.resolve("config"); - final Path scriptPath = configPath.resolve("scripts"); - try { - Files.createDirectories(scriptPath); - } catch (IOException e) { - throw new RuntimeException("failed to create config dir"); - - } - try (InputStream stream = SearchInputIT.class.getResourceAsStream("/org/elasticsearch/xpack/watcher/input/search/config/scripts" + - "/test_disk_template.mustache"); - OutputStream out = Files.newOutputStream(scriptPath.resolve("test_disk_template.mustache"))) { - Streams.copy(stream, out); - } catch (IOException e) { - throw new RuntimeException("failed to copy mustache template"); - } - - - //Set path so ScriptService will pick up the test scripts - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put("path.conf", configPath).build(); - } - - @Override - protected Settings transportClientSettings() { - return Settings.builder() - .put(super.transportClientSettings()) - .build(); - } - - public void testExecute() throws Exception { - SearchSourceBuilder searchSourceBuilder = searchSource().query( - boolQuery().must(matchQuery("event_type", "a")).must(rangeQuery("_timestamp") - .from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}"))); - SearchRequest searchRequest = client() - .prepareSearch() - .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) - .request() - .source(searchSourceBuilder); - - WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(searchRequest); - ExecutableSearchInput searchInput = new ExecutableSearchInput(new SearchInput(request, null, null, null), logger, - WatcherClientProxy.of(client()), watcherSearchTemplateService(), null); - WatchExecutionContext ctx = new TriggeredExecutionContext( - new Watch("test-watch", - new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))), - new ExecutableSimpleInput(new SimpleInput(new Payload.Simple()), logger), - new ExecutableAlwaysCondition(logger), - null, - null, - new ExecutableActions(new ArrayList()), - null, - new WatchStatus(new DateTime(0, UTC), emptyMap())), - new DateTime(0, UTC), - new ScheduleTriggerEvent("test-watch", new DateTime(0, UTC), new DateTime(0, UTC)), - timeValueSeconds(5)); - SearchInput.Result result = searchInput.execute(ctx, new Payload.Simple()); - - assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); - assertNotNull(result.executedRequest()); - assertThat(result.status(), is(Input.Result.Status.SUCCESS)); - assertEquals(result.executedRequest().searchType(), request.getRequest().searchType()); - assertArrayEquals(result.executedRequest().indices(), request.getRequest().indices()); - assertEquals(result.executedRequest().indicesOptions(), request.getRequest().indicesOptions()); - - XContentSource source = toXContentSource(result); - assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:00:00.000Z||-30s")); - assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:00:00.000Z")); - } - - public void testSearchInlineTemplate() throws Exception { - WatchExecutionContext ctx = createContext(); - - Map triggerParams = new HashMap(); - triggerParams.put("triggered_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); - triggerParams.put("scheduled_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); - Map ctxParams = new HashMap(); - ctxParams.put("id", ctx.id().value()); - ctxParams.put("metadata", null); - ctxParams.put("vars", new HashMap()); - ctxParams.put("watch_id", "test-watch"); - ctxParams.put("trigger", triggerParams); - ctxParams.put("payload", new Payload.Simple().data()); - ctxParams.put("execution_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); - Map expectedParams = new HashMap(); - expectedParams.put("seconds_param", "30s"); - expectedParams.put("ctx", ctxParams); - Map params = new HashMap<>(); - params.put("seconds_param", "30s"); - - WatcherScript template = WatcherScript.inline(TEMPLATE_QUERY).lang("mustache").params(params).build(); - - SearchRequest request = client().prepareSearch() - .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").request(); - - SearchInput.Result executedResult = executeSearchInput(request, template, ctx); - - assertNotNull(executedResult.executedRequest()); - assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS)); - if (getNumShards("test-search-index").numPrimaries > 1) { - assertEquals(executedResult.executedRequest().searchType(), request.searchType()); - } - assertArrayEquals(executedResult.executedRequest().indices(), request.indices()); - assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions()); - - XContentSource source = toXContentSource(executedResult); - assertThat(source.getValue("query.bool.filter.0.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); - assertThat(source.getValue("query.bool.filter.0.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); - } - - public void testSearchIndexedTemplate() throws Exception { - WatchExecutionContext ctx = createContext(); - - PutStoredScriptRequest indexedScriptRequest = client().admin().cluster().preparePutStoredScript() - .setId("test-template") - .setScriptLang("mustache") - .setSource(new BytesArray(TEMPLATE_QUERY)) - .request(); - assertThat(client().admin().cluster().putStoredScript(indexedScriptRequest).actionGet().isAcknowledged(), is(true)); - - Map params = new HashMap<>(); - params.put("seconds_param", "30s"); - - WatcherScript template = WatcherScript.indexed("test-template").lang("mustache").params(params).build(); - - jsonBuilder().value(TextTemplate.indexed("test-template").params(params).build()).bytes(); - SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").request(); - - SearchInput.Result executedResult = executeSearchInput(request, template, ctx); - - assertNotNull(executedResult.executedRequest()); - assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS)); - if (getNumShards("test-search-index").numPrimaries > 1) { - assertEquals(executedResult.executedRequest().searchType(), request.searchType()); - } - assertArrayEquals(executedResult.executedRequest().indices(), request.indices()); - assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions()); - - XContentSource source = toXContentSource(executedResult); - assertThat(source.getValue("query.bool.filter.0.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); - assertThat(source.getValue("query.bool.filter.0.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); - - } - - public void testSearchOnDiskTemplate() throws Exception { - WatchExecutionContext ctx = createContext(); - - Map params = new HashMap<>(); - params.put("seconds_param", "30s"); - - WatcherScript template = WatcherScript.file("test_disk_template").lang("mustache").params(params).build(); - SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").request(); - - SearchInput.Result executedResult = executeSearchInput(request, template, ctx); - - assertNotNull(executedResult.executedRequest()); - assertThat(executedResult.status(), is(Input.Result.Status.SUCCESS)); - assertArrayEquals(executedResult.executedRequest().indices(), request.indices()); - assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions()); - } - - public void testDifferentSearchType() throws Exception { - SearchSourceBuilder searchSourceBuilder = searchSource().query( - boolQuery().must(matchQuery("event_type", "a")).must(rangeQuery("_timestamp") - .from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}")) - ); - SearchType searchType = getRandomSupportedSearchType(); - - SearchRequest searchRequest = client() - .prepareSearch() - .setSearchType(searchType) - .request() - .source(searchSourceBuilder); - - WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(searchRequest); - - ExecutableSearchInput searchInput = new ExecutableSearchInput(new SearchInput(request, null, null, null), logger, - WatcherClientProxy.of(client()), watcherSearchTemplateService(), null); - WatchExecutionContext ctx = new TriggeredExecutionContext( - new Watch("test-watch", - new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))), - new ExecutableSimpleInput(new SimpleInput(new Payload.Simple()), logger), - new ExecutableAlwaysCondition(logger), - null, - null, - new ExecutableActions(new ArrayList()), - null, - new WatchStatus(new DateTime(0, UTC), emptyMap())), - new DateTime(0, UTC), - new ScheduleTriggerEvent("test-watch", new DateTime(0, UTC), new DateTime(0, UTC)), - timeValueSeconds(5)); - SearchInput.Result result = searchInput.execute(ctx, new Payload.Simple()); - - assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); - assertNotNull(result.executedRequest()); - assertThat(result.status(), is(Input.Result.Status.SUCCESS)); - assertEquals(result.executedRequest().searchType(), searchType); - assertArrayEquals(result.executedRequest().indices(), searchRequest.indices()); - assertEquals(result.executedRequest().indicesOptions(), searchRequest.indicesOptions()); - - XContentSource source = toXContentSource(result); - assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:00:00.000Z||-30s")); - assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:00:00.000Z")); - } - - public void testParserValid() throws Exception { - SearchRequest searchRequest = client().prepareSearch() - .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) - .request() - .source(searchSource() - .query(boolQuery().must(matchQuery("event_type", "a")).must(rangeQuery("_timestamp") - .from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}")))); - - TimeValue timeout = randomBoolean() ? TimeValue.timeValueSeconds(randomInt(10)) : null; - XContentBuilder builder = jsonBuilder().value( - new SearchInput(new WatcherSearchTemplateRequest(searchRequest), null, timeout, null)); - XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes()); - parser.nextToken(); - - IndicesQueriesRegistry indicesQueryRegistry = internalCluster().getInstance(IndicesQueriesRegistry.class); - SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, WatcherClientProxy.of(client()), indicesQueryRegistry, - null, null, scriptService()); - - SearchInput searchInput = factory.parseInput("_id", parser); - assertEquals(SearchInput.TYPE, searchInput.type()); - assertThat(searchInput.getTimeout(), equalTo(timeout)); - } - - private WatchExecutionContext createContext() { - return new TriggeredExecutionContext( - new Watch("test-watch", - new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))), - new ExecutableSimpleInput(new SimpleInput(new Payload.Simple()), logger), - new ExecutableAlwaysCondition(logger), - null, - null, - new ExecutableActions(new ArrayList()), - null, - new WatchStatus(new DateTime(50000, UTC), emptyMap())), - new DateTime(60000, UTC), - new ScheduleTriggerEvent("test-watch", new DateTime(60000, UTC), new DateTime(60000, UTC)), - timeValueSeconds(5)); - } - - private SearchInput.Result executeSearchInput(SearchRequest request, WatcherScript template, - WatchExecutionContext ctx) throws IOException { - createIndex("test-search-index"); - ensureGreen("test-search-index"); - SearchInput.Builder siBuilder = SearchInput.builder(new WatcherSearchTemplateRequest(request, template)); - - SearchInput si = siBuilder.build(); - - ExecutableSearchInput searchInput = new ExecutableSearchInput(si, logger, WatcherClientProxy.of(client()), - watcherSearchTemplateService(), null); - return searchInput.execute(ctx, new Payload.Simple()); - } - - protected WatcherSearchTemplateService watcherSearchTemplateService() { - String master = internalCluster().getMasterName(); - return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(), - internalCluster().getInstance(ScriptService.class, master), - internalCluster().getInstance(IndicesQueriesRegistry.class, master), - internalCluster().getInstance(AggregatorParsers.class, master), - internalCluster().getInstance(Suggesters.class, master) - ); - } - - protected ScriptService scriptService() { - return internalCluster().getInstance(ScriptService.class); - } - - private XContentSource toXContentSource(SearchInput.Result result) throws IOException { - try (XContentBuilder builder = jsonBuilder()) { - result.executedRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS); - return new XContentSource(builder); - } - } - - /** - * Custom plugin that registers XPack script context. - */ - public static class CustomScriptContextPlugin extends Plugin implements ScriptPlugin { - - @Override - public ScriptContext.Plugin getCustomScriptContexts() { - return WatcherScript.CTX_PLUGIN; - } - } -} diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java deleted file mode 100644 index c6429e0f4df..00000000000 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -/** - * This package contains tests that use mustache to test what looks - * to be unrelated functionality, or functionality that should be - * tested with a mock instead. Instead of doing an epic battle - * with these tests, they are temporarily moved here to the mustache - * module's tests, but that is likely not where they belong. Please - * help by cleaning them up and we can remove this package! - * - *
    - *
  • If the test is testing templating integration with another core subsystem, - * fix it to use a mock instead, so it can be in the core tests again
    • - *
  • If the test is just being lazy, and does not really need templating to test - * something, clean it up!
    • - *
- */ - -package org.elasticsearch.messy.tests; diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/resources/org/elasticsearch/xpack/watcher/input/search/config/scripts/test_disk_template.mustache b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/resources/org/elasticsearch/xpack/watcher/input/search/config/scripts/test_disk_template.mustache deleted file mode 100644 index cdc73453c0a..00000000000 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/resources/org/elasticsearch/xpack/watcher/input/search/config/scripts/test_disk_template.mustache +++ /dev/null @@ -1,26 +0,0 @@ -{ - "query": { - "bool": { - "must" : [ - { - "match": { - "event_type": { - "query": "a", - "type": "boolean" - } - } - }, - { - "range": { - "_timestamp": { - "from": "{{ctx.trigger.scheduled_time}}||-{{seconds_param}}", - "to": "{{ctx.trigger.scheduled_time}}", - "include_lower": true, - "include_upper": true - } - } - } - ] - } - } -} \ No newline at end of file diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/resources/org/elasticsearch/xpack/watcher/transform/search/config/scripts/test_disk_template.mustache b/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/resources/org/elasticsearch/xpack/watcher/transform/search/config/scripts/test_disk_template.mustache deleted file mode 100644 index cdc73453c0a..00000000000 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/resources/org/elasticsearch/xpack/watcher/transform/search/config/scripts/test_disk_template.mustache +++ /dev/null @@ -1,26 +0,0 @@ -{ - "query": { - "bool": { - "must" : [ - { - "match": { - "event_type": { - "query": "a", - "type": "boolean" - } - } - }, - { - "range": { - "_timestamp": { - "from": "{{ctx.trigger.scheduled_time}}||-{{seconds_param}}", - "to": "{{ctx.trigger.scheduled_time}}", - "include_lower": true, - "include_upper": true - } - } - } - ] - } - } -} \ No newline at end of file diff --git a/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/30_search_input_and_transform.yaml b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/30_search_input_and_transform.yaml new file mode 100644 index 00000000000..a8c7c9308bc --- /dev/null +++ b/elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/30_search_input_and_transform.yaml @@ -0,0 +1,157 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + - do: {xpack.watcher.stats:{}} + - do: + index: + index: idx + type: type + id: 1 + body: > + { + "date" : "2015-01-01T00:00:00", + "value" : "val_1" + } + - do: + index: + index: idx + type: type + id: 2 + body: > + { + "date" : "2015-01-02T00:00:00", + "value" : "val_2" + } + - do: + index: + index: idx + type: type + id: 3 + body: > + { + "date" : "2015-01-03T00:00:00", + "value" : "val_3" + } + - do: + index: + index: idx + type: type + id: 4 + body: > + { + "date" : "2015-01-04T00:00:00", + "value" : "val_4" + } + - do: + indices.refresh: + index: idx + +--- +"Test input mustache integration": + - do: + xpack.watcher.execute_watch: + body: > + { + "trigger_data" : { + "scheduled_time" : "2015-01-04T00:00:00" + }, + "watch" : { + "trigger" : { "schedule" : { "interval" : "10s" } }, + "actions" : { + "dummy" : { + "logging" : { + "text" : "executed!" + } + } + }, + "input" : { + "search" : { + "request" : { + "indices" : "idx", + "body" : { + "query" : { + "bool" : { + "filter" : [ + { + "range" : { + "date" : { + "lte" : "{{ctx.trigger.scheduled_time}}", + "gte" : "{{ctx.trigger.scheduled_time}}||-3d" + } + } + } + ] + } + } + } + } + } + } + } + } + - match: { "watch_record.result.input.type": "search" } + - match: { "watch_record.result.input.status": "success" } + - match: { "watch_record.result.input.payload.hits.total": 4 } + # makes sure that the mustache template snippets have been resolved correctly: + - match: { "watch_record.result.input.search.request.body.query.bool.filter.0.range.date.from": "2015-01-04T00:00:00.000Z||-3d" } + - match: { "watch_record.result.input.search.request.body.query.bool.filter.0.range.date.to": "2015-01-04T00:00:00.000Z" } + +--- +"Test transform mustache integration": + - do: + xpack.watcher.execute_watch: + body: > + { + "trigger_data" : { + "scheduled_time" : "2015-01-04T00:00:00" + }, + "watch" : { + "trigger" : { "schedule" : { "interval" : "10s" } }, + "input" : { "simple" : { "value" : "val_3" } }, + "actions" : { + "dummy" : { + "logging" : { + "text" : "executed!" + } + } + }, + "transform" : { + "search" : { + "request" : { + "indices" : "idx", + "body" : { + "query" : { + "bool" : { + "filter" : [ + { + "range" : { + "date" : { + "lte" : "{{ctx.trigger.scheduled_time}}", + "gte" : "{{ctx.trigger.scheduled_time}}||-1d" + } + } + }, + { + "term" : { + "value" : "{{ctx.payload.value}}" + } + } + ] + } + } + } + } + } + } + } + } + - match: { "watch_record.result.transform.type": "search" } + - match: { "watch_record.result.transform.status": "success" } + - match: { "watch_record.result.transform.payload.hits.total": 1 } + - match: { "watch_record.result.transform.payload.hits.hits.0._id": "3" } + # makes sure that the mustache template snippets have been resolved correctly: + - match: { "watch_record.result.transform.search.request.body.query.bool.filter.0.range.date.from": "2015-01-04T00:00:00.000Z||-1d" } + - match: { "watch_record.result.transform.search.request.body.query.bool.filter.0.range.date.to": "2015-01-04T00:00:00.000Z" } + - match: { "watch_record.result.transform.search.request.body.query.bool.filter.1.term.value.value": "val_3" } diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/SecurityCachePermissionTests.java similarity index 53% rename from elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java rename to elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/SecurityCachePermissionTests.java index f84c4cc74ad..0b9511a5f1a 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SecurityCachePermissionIT.java +++ b/elasticsearch/x-pack/security/src/test/java/org/elasticsearch/integration/SecurityCachePermissionTests.java @@ -3,43 +3,26 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.integration; import org.elasticsearch.ElasticsearchSecurityException; -import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.TermsLookup; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.mustache.MustachePlugin; -import org.elasticsearch.script.mustache.TemplateQueryBuilder; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.xpack.security.authc.support.SecuredString; import org.junit.Before; -import org.junit.BeforeClass; - -import java.util.ArrayList; -import java.util.Collection; import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; -@SecurityIntegTestCase.AwaitsFix(bugUrl = "clean up test to not use mustache templates, otherwise needs many resources here") -public class SecurityCachePermissionIT extends SecurityIntegTestCase { - static final String READ_ONE_IDX_USER = "read_user"; - - @Override - protected Collection> nodePlugins() { - Collection> types = new ArrayList<>(); - types.addAll(super.nodePlugins()); - types.add(MustachePlugin.class); - return types; - } +public class SecurityCachePermissionTests extends SecurityIntegTestCase { + private final String READ_ONE_IDX_USER = "read_user"; + @Override public String configUsers() { return super.configUsers() @@ -61,11 +44,6 @@ public class SecurityCachePermissionIT extends SecurityIntegTestCase { + "read_one_idx:" + READ_ONE_IDX_USER + "\n"; } - @BeforeClass - public static void checkVersion() { - assumeTrue("These tests are only valid with elasticsearch 1.6.0+", Version.CURRENT.id >= 1060099); - } - @Before public void loadData() { index("data", "a", "1", "{ \"name\": \"John\", \"token\": \"token1\" }"); @@ -93,33 +71,4 @@ public class SecurityCachePermissionIT extends SecurityIntegTestCase { assertThat(e.toString(), containsString("unauthorized")); } } - - public void testThatScriptServiceDoesntLeakData() { - String source = "{\n" + - "\"template\": {\n" + - " \"query\": {\n" + - " \"exists\": {\n" + - " \"field\": \"{{name}}\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - - //Template template = new Template(source, INLINE, MustacheScriptEngineService.NAME, null, singletonMap("name", "token")); - SearchResponse response = client().prepareSearch("data").setTypes("a") - .setQuery(new TemplateQueryBuilder(source, ScriptService.ScriptType.INLINE, singletonMap("name", "token"))) - .execute().actionGet(); - assertThat(response.isTimedOut(), is(false)); - assertThat(response.getHits().hits().length, is(1)); - - // Repeat with unauthorized user!!!! - ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> client() - .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER, - new SecuredString("changeme".toCharArray())))) - .prepareSearch("data").setTypes("a") - .setQuery(new TemplateQueryBuilder(source, ScriptService.ScriptType.INLINE, singletonMap("name", "token"))) - .execute().actionGet()); - assertThat(e.toString(), containsString("ElasticsearchSecurityException[action")); - assertThat(e.toString(), containsString("unauthorized")); - } } diff --git a/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java new file mode 100644 index 00000000000..aa5c721369f --- /dev/null +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java @@ -0,0 +1,199 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.watcher.test.integration; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.MockMustacheScriptEngine; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.suggest.Suggesters; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.xpack.watcher.actions.ExecutableActions; +import org.elasticsearch.xpack.watcher.condition.always.ExecutableAlwaysCondition; +import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext; +import org.elasticsearch.xpack.watcher.execution.WatchExecutionContext; +import org.elasticsearch.xpack.watcher.input.Input; +import org.elasticsearch.xpack.watcher.input.search.ExecutableSearchInput; +import org.elasticsearch.xpack.watcher.input.search.SearchInput; +import org.elasticsearch.xpack.watcher.input.search.SearchInputFactory; +import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput; +import org.elasticsearch.xpack.watcher.input.simple.SimpleInput; +import org.elasticsearch.xpack.watcher.support.WatcherScript; +import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; +import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; +import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; +import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger; +import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent; +import org.elasticsearch.xpack.watcher.watch.Payload; +import org.elasticsearch.xpack.watcher.watch.Watch; +import org.elasticsearch.xpack.watcher.watch.WatchStatus; +import org.joda.time.DateTime; + +import java.util.ArrayList; +import java.util.Collection; + +import static java.util.Collections.emptyMap; +import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; +import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; +import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.joda.time.DateTimeZone.UTC; + +@ClusterScope(scope = SUITE, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, supportsDedicatedMasters = false, + numDataNodes = 1) +public class SearchInputTests extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + Collection> types = new ArrayList<>(); + types.addAll(super.nodePlugins()); + types.add(MockMustacheScriptEngine.TestPlugin.class); + types.add(CustomScriptContextPlugin.class); + return types; + } + + public void testExecute() throws Exception { + SearchSourceBuilder searchSourceBuilder = searchSource().query( + boolQuery().must(matchQuery("event_type", "a"))); + SearchRequest searchRequest = client() + .prepareSearch() + .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) + .request() + .source(searchSourceBuilder); + + WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(searchRequest); + ExecutableSearchInput searchInput = new ExecutableSearchInput(new SearchInput(request, null, null, null), logger, + WatcherClientProxy.of(client()), watcherSearchTemplateService(), null); + WatchExecutionContext ctx = new TriggeredExecutionContext( + new Watch("test-watch", + new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))), + new ExecutableSimpleInput(new SimpleInput(new Payload.Simple()), logger), + new ExecutableAlwaysCondition(logger), + null, + null, + new ExecutableActions(new ArrayList<>()), + null, + new WatchStatus(new DateTime(0, UTC), emptyMap())), + new DateTime(0, UTC), + new ScheduleTriggerEvent("test-watch", new DateTime(0, UTC), new DateTime(0, UTC)), + timeValueSeconds(5)); + SearchInput.Result result = searchInput.execute(ctx, new Payload.Simple()); + + assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); + assertNotNull(result.executedRequest()); + assertThat(result.status(), is(Input.Result.Status.SUCCESS)); + assertEquals(result.executedRequest().searchType(), request.getRequest().searchType()); + assertArrayEquals(result.executedRequest().indices(), request.getRequest().indices()); + assertEquals(result.executedRequest().indicesOptions(), request.getRequest().indicesOptions()); + } + + public void testDifferentSearchType() throws Exception { + SearchSourceBuilder searchSourceBuilder = searchSource().query( + boolQuery().must(matchQuery("event_type", "a")) + ); + SearchType searchType = getRandomSupportedSearchType(); + + SearchRequest searchRequest = client() + .prepareSearch() + .setSearchType(searchType) + .request() + .source(searchSourceBuilder); + + WatcherSearchTemplateRequest request = new WatcherSearchTemplateRequest(searchRequest); + + ExecutableSearchInput searchInput = new ExecutableSearchInput(new SearchInput(request, null, null, null), logger, + WatcherClientProxy.of(client()), watcherSearchTemplateService(), null); + WatchExecutionContext ctx = new TriggeredExecutionContext( + new Watch("test-watch", + new ScheduleTrigger(new IntervalSchedule(new IntervalSchedule.Interval(1, IntervalSchedule.Interval.Unit.MINUTES))), + new ExecutableSimpleInput(new SimpleInput(new Payload.Simple()), logger), + new ExecutableAlwaysCondition(logger), + null, + null, + new ExecutableActions(new ArrayList<>()), + null, + new WatchStatus(new DateTime(0, UTC), emptyMap())), + new DateTime(0, UTC), + new ScheduleTriggerEvent("test-watch", new DateTime(0, UTC), new DateTime(0, UTC)), + timeValueSeconds(5)); + SearchInput.Result result = searchInput.execute(ctx, new Payload.Simple()); + + assertThat(XContentMapValues.extractValue("hits.total", result.payload().data()), equalTo(0)); + assertNotNull(result.executedRequest()); + assertThat(result.status(), is(Input.Result.Status.SUCCESS)); + assertEquals(result.executedRequest().searchType(), searchType); + assertArrayEquals(result.executedRequest().indices(), searchRequest.indices()); + assertEquals(result.executedRequest().indicesOptions(), searchRequest.indicesOptions()); + } + + public void testParserValid() throws Exception { + SearchRequest searchRequest = client().prepareSearch() + .setSearchType(ExecutableSearchInput.DEFAULT_SEARCH_TYPE) + .request() + .source(searchSource() + .query(boolQuery().must(matchQuery("event_type", "a")).must(rangeQuery("_timestamp") + .from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}")))); + + TimeValue timeout = randomBoolean() ? TimeValue.timeValueSeconds(randomInt(10)) : null; + XContentBuilder builder = jsonBuilder().value( + new SearchInput(new WatcherSearchTemplateRequest(searchRequest), null, timeout, null)); + XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes()); + parser.nextToken(); + + IndicesQueriesRegistry indicesQueryRegistry = internalCluster().getInstance(IndicesQueriesRegistry.class); + SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, WatcherClientProxy.of(client()), indicesQueryRegistry, + null, null, scriptService()); + + SearchInput searchInput = factory.parseInput("_id", parser); + assertEquals(SearchInput.TYPE, searchInput.type()); + assertThat(searchInput.getTimeout(), equalTo(timeout)); + } + + private WatcherSearchTemplateService watcherSearchTemplateService() { + String master = internalCluster().getMasterName(); + return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(), + internalCluster().getInstance(ScriptService.class, master), + internalCluster().getInstance(IndicesQueriesRegistry.class, master), + internalCluster().getInstance(AggregatorParsers.class, master), + internalCluster().getInstance(Suggesters.class, master) + ); + } + + private ScriptService scriptService() { + return internalCluster().getInstance(ScriptService.class); + } + + /** + * Custom plugin that registers XPack script context. + */ + public static class CustomScriptContextPlugin extends Plugin implements ScriptPlugin { + + @Override + public ScriptContext.Plugin getCustomScriptContexts() { + return WatcherScript.CTX_PLUGIN; + } + } +} diff --git a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java similarity index 53% rename from elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java rename to elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java index c45db63abb7..b09a5c85b76 100644 --- a/elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/SearchTransformIT.java +++ b/elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java @@ -3,15 +3,12 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.messy.tests; +package org.elasticsearch.xpack.watcher.test.integration; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Requests; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; @@ -24,9 +21,9 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.MockMustacheScriptEngine; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.Suggesters; @@ -43,7 +40,6 @@ import org.elasticsearch.xpack.watcher.support.WatcherScript; import org.elasticsearch.xpack.watcher.support.init.proxy.WatcherClientProxy; import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest; import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService; -import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.watcher.transform.Transform; import org.elasticsearch.xpack.watcher.transform.TransformBuilders; import org.elasticsearch.xpack.watcher.transform.search.ExecutableSearchTransform; @@ -56,35 +52,24 @@ import org.elasticsearch.xpack.watcher.watch.Payload; import org.elasticsearch.xpack.watcher.watch.Watch; import org.elasticsearch.xpack.watcher.watch.WatchStatus; import org.joda.time.DateTime; -import org.joda.time.chrono.ISOChronology; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Base64; import java.util.Collection; -import java.util.HashMap; import java.util.Map; import static java.util.Collections.emptyMap; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; -import static org.elasticsearch.xpack.watcher.support.WatcherDateTimeUtils.parseDate; import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.EMPTY_PAYLOAD; import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.getRandomSupportedSearchType; import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.mockExecutionContext; -import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.simplePayload; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -94,63 +79,26 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.joda.time.DateTimeZone.UTC; -/** - * - */ @ClusterScope(scope = SUITE, numClientNodes = 0, transportClientRatio = 0, randomDynamicTemplates = false, supportsDedicatedMasters = false, numDataNodes = 1) -public class SearchTransformIT extends ESIntegTestCase { +public class SearchTransformTests extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - Collection> types = new ArrayList<>(); - types.addAll(super.nodePlugins()); - types.add(MustachePlugin.class); - types.add(CustomScriptContextPlugin.class); - return types; - } - - @Override - public Settings nodeSettings(int nodeOrdinal) { - final Path tempDir = createTempDir(); - final Path configPath = tempDir.resolve("config"); - final Path scriptPath = configPath.resolve("scripts"); - try { - Files.createDirectories(scriptPath); - } catch (IOException e) { - throw new RuntimeException("failed to create config dir"); - - } - String path = "/org/elasticsearch/xpack/watcher/transform/search/config/scripts/test_disk_template.mustache"; - try (InputStream stream = SearchTransformIT.class.getResourceAsStream(path); - OutputStream out = Files.newOutputStream(scriptPath.resolve("test_disk_template.mustache"))) { - Streams.copy(stream, out); - } catch (IOException e) { - throw new RuntimeException("failed to copy mustache template"); - } - //Set path so ScriptService will pick up the test scripts - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - // we're not extending from the base watcher test case, so we should prevent the watcher plugin from being loaded - .put("path.conf", configPath).build(); - } - - @Override - protected Settings transportClientSettings() { - return Settings.builder() - .put(super.transportClientSettings()) - .build(); + Collection> plugins = new ArrayList<>(); + plugins.addAll(super.nodePlugins()); + plugins.add(CustomScriptContextPlugin.class); + plugins.add(MockMustacheScriptEngine.TestPlugin.class); + return plugins; } @Override public Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) - // we have to test this on an index that has at least 2 shards. Otherwise when searching indices with // a single shard the QUERY_THEN_FETCH search type will change to QUERY_AND_FETCH during execution. .put("index.number_of_shards", randomIntBetween(2, 5)) - .build(); } @@ -232,71 +180,6 @@ public class SearchTransformIT extends ESIntegTestCase { } } - public void testExecuteMustacheTemplate() throws Exception { - - // The rational behind this test: - // - // - we index 4 documents each one associated with a unique value and each is associated with a day - // - we build a search transform such that with a filter that - // - the date must be after [scheduled_time] variable - // - the date must be before [execution_time] variable - // - the value must match [payload.value] variable - // - the variable are set as such: - // - scheduled_time = youngest document's date - // - fired_time = oldest document's date - // - payload.value = val_3 - // - when executed, the variables will be replaced with the scheduled_time, fired_time and the payload.value. - // - we set all these variables accordingly (the search transform is responsible to populate them) - // - when replaced correctly, the search should return document 3. - // - // we then do a search for document 3, and compare the response to the payload returned by the transform - - index("idx", "type", "1", doc("2015-01-01T00:00:00", "val_1")); - index("idx", "type", "2", doc("2015-01-02T00:00:00", "val_2")); - index("idx", "type", "3", doc("2015-01-03T00:00:00", "val_3")); - index("idx", "type", "4", doc("2015-01-04T00:00:00", "val_4")); - - ensureGreen("idx"); - refresh(); - - SearchRequest request = Requests.searchRequest("idx").source(searchSource().query(boolQuery() - .must(constantScoreQuery(rangeQuery("date").gt("{{ctx.trigger.scheduled_time}}"))) - .must(constantScoreQuery(rangeQuery("date").lt("{{ctx.execution_time}}"))) - .must(termQuery("value", "{{ctx.payload.value}}")))); - - SearchTransform searchTransform = TransformBuilders.searchTransform(request).build(); - ExecutableSearchTransform transform = new ExecutableSearchTransform(searchTransform, logger, WatcherClientProxy.of(client()), - watcherSearchTemplateService(), null); - - ScheduleTriggerEvent event = new ScheduleTriggerEvent("_name", parseDate("2015-01-04T00:00:00", UTC), - parseDate("2015-01-01T00:00:00", UTC)); - WatchExecutionContext ctx = mockExecutionContext("_name", parseDate("2015-01-04T00:00:00", UTC), event, EMPTY_PAYLOAD); - - Payload payload = simplePayload("value", "val_3"); - - Transform.Result result = transform.execute(ctx, payload); - assertThat(result, notNullValue()); - assertThat(result.type(), is(SearchTransform.TYPE)); - - SearchResponse response = client().prepareSearch("idx").setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE).setQuery( - boolQuery() - .must(constantScoreQuery(rangeQuery("date").gt(parseDate("2015-01-01T00:00:00", UTC)))) - .must(constantScoreQuery(rangeQuery("date").lt(parseDate("2015-01-04T00:00:00", UTC)))) - .must(termQuery("value", "val_3")) - ).get(); - Payload expectedPayload = new Payload.XContent(response); - - // we need to remove the "took" field from teh response as this is the only field - // that most likely be different between the two... we don't really care about this - // field, we just want to make sure that the important parts of the response are the same - Map resultData = result.payload().data(); - resultData.remove("took"); - Map expectedData = expectedPayload.data(); - expectedData.remove("took"); - - assertThat(resultData, equalTo(expectedData)); - } - public void testParser() throws Exception { String[] indices = rarely() ? null : randomBoolean() ? new String[] { "idx" } : new String[] { "idx1", "idx2" }; SearchType searchType = getRandomSupportedSearchType(); @@ -354,116 +237,11 @@ public class SearchTransformIT extends ESIntegTestCase { assertThat(executable.transform().getTimeout(), equalTo(readTimeout)); } - public void testSearchInlineTemplate() throws Exception { - WatchExecutionContext ctx = createContext(); - - final String templateQuery = "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"event_type\":{\"query\":\"a\"," + - "\"type\":\"boolean\"}}},{\"range\":{\"_timestamp\":" + - "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," + - "\"include_lower\":true,\"include_upper\":true}}}]}}}"; - - Map triggerParams = new HashMap(); - triggerParams.put("triggered_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); - triggerParams.put("scheduled_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); - Map ctxParams = new HashMap(); - ctxParams.put("id", ctx.id().value()); - ctxParams.put("metadata", null); - ctxParams.put("vars", new HashMap()); - ctxParams.put("watch_id", "test-watch"); - ctxParams.put("payload", new HashMap()); - ctxParams.put("trigger", triggerParams); - ctxParams.put("execution_time", new DateTime(1970, 01, 01, 00, 01, 00, 000, ISOChronology.getInstanceUTC())); - Map expectedParams = new HashMap(); - expectedParams.put("seconds_param", "30s"); - expectedParams.put("ctx", ctxParams); - - Map params = new HashMap<>(); - params.put("seconds_param", "30s"); - - WatcherScript template = WatcherScript.inline(templateQuery).lang("mustache").params(params).build(); - SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").request(); - - SearchTransform.Result executedResult = executeSearchTransform(request, template, ctx); - - assertThat(executedResult.status(), is(Transform.Result.Status.SUCCESS)); - assertEquals(executedResult.executedRequest().searchType(), request.searchType()); - assertArrayEquals(executedResult.executedRequest().indices(), request.indices()); - assertEquals(executedResult.executedRequest().indicesOptions(), request.indicesOptions()); - - XContentSource source = toXContentSource(executedResult); - assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); - assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); - } - - public void testSearchIndexedTemplate() throws Exception { - WatchExecutionContext ctx = createContext(); - - final String templateQuery = "{\"query\":{\"bool\":{\"must\":[{\"match\":{\"event_type\":{\"query\":\"a\"," + - "\"type\":\"boolean\"}}},{\"range\":{\"_timestamp\":" + - "{\"from\":\"{{ctx.trigger.scheduled_time}}||-{{seconds_param}}\",\"to\":\"{{ctx.trigger.scheduled_time}}\"," + - "\"include_lower\":true,\"include_upper\":true}}}]}}}"; - - PutStoredScriptRequest indexedScriptRequest = client().admin().cluster().preparePutStoredScript() - .setId("test-script") - .setScriptLang("mustache") - .setSource(new BytesArray(templateQuery)) - .request(); - assertThat(client().admin().cluster().putStoredScript(indexedScriptRequest).actionGet().isAcknowledged(), is(true)); - - Map params = new HashMap<>(); - params.put("seconds_param", "30s"); - - WatcherScript template = WatcherScript.indexed("test-script").lang("mustache").params(params).build(); - - SearchRequest request = client() - .prepareSearch() - .setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index") - .request(); - - SearchTransform.Result result = executeSearchTransform(request, template, ctx); - - assertNotNull(result.executedRequest()); - assertThat(result.status(), is(Transform.Result.Status.SUCCESS)); - assertArrayEquals(result.executedRequest().indices(), request.indices()); - assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions()); - - XContentSource source = toXContentSource(result); - assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); - assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); - } - - public void testSearchOnDiskTemplate() throws Exception { - WatchExecutionContext ctx = createContext(); - - Map params = new HashMap<>(); - params.put("seconds_param", "30s"); - - WatcherScript template = WatcherScript.file("test_disk_template").lang("mustache").params(params).build(); - SearchRequest request = client().prepareSearch().setSearchType(ExecutableSearchTransform.DEFAULT_SEARCH_TYPE) - .setIndices("test-search-index").request(); - - SearchTransform.Result result = executeSearchTransform(request, template, ctx); - - assertNotNull(result.executedRequest()); - assertThat(result.status(), is(Transform.Result.Status.SUCCESS)); - assertArrayEquals(result.executedRequest().indices(), request.indices()); - assertEquals(result.executedRequest().indicesOptions(), request.indicesOptions()); - - XContentSource source = toXContentSource(result); - assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); - assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); - } - public void testDifferentSearchType() throws Exception { WatchExecutionContext ctx = createContext(); SearchSourceBuilder searchSourceBuilder = searchSource().query(boolQuery() - .must(matchQuery("event_type", "a")) - .must(rangeQuery("_timestamp") - .from("{{ctx.trigger.scheduled_time}}||-30s") - .to("{{ctx.trigger.triggered_time}}"))); + .must(matchQuery("event_type", "a"))); final SearchType searchType = getRandomSupportedSearchType(); SearchRequest request = client() @@ -480,10 +258,6 @@ public class SearchTransformIT extends ESIntegTestCase { assertThat(result.executedRequest().searchType(), is(searchType)); assertThat(result.executedRequest().indices(), arrayContainingInAnyOrder(request.indices())); assertThat(result.executedRequest().indicesOptions(), equalTo(request.indicesOptions())); - - XContentSource source = toXContentSource(result); - assertThat(source.getValue("query.bool.must.1.range._timestamp.from"), equalTo("1970-01-01T00:01:00.000Z||-30s")); - assertThat(source.getValue("query.bool.must.1.range._timestamp.to"), equalTo("1970-01-01T00:01:00.000Z")); } private WatchExecutionContext createContext() { @@ -515,7 +289,7 @@ public class SearchTransformIT extends ESIntegTestCase { return executableSearchTransform.execute(ctx, Payload.Simple.EMPTY); } - protected WatcherSearchTemplateService watcherSearchTemplateService() { + private WatcherSearchTemplateService watcherSearchTemplateService() { String master = internalCluster().getMasterName(); return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(), internalCluster().getInstance(ScriptService.class, master), @@ -525,24 +299,10 @@ public class SearchTransformIT extends ESIntegTestCase { ); } - protected ScriptService scriptService() { + private ScriptService scriptService() { return internalCluster().getInstance(ScriptService.class); } - private static Map doc(String date, String value) { - Map doc = new HashMap<>(); - doc.put("date", parseDate(date, UTC)); - doc.put("value", value); - return doc; - } - - private XContentSource toXContentSource(SearchTransform.Result result) throws IOException { - try (XContentBuilder builder = jsonBuilder()) { - result.executedRequest().source().toXContent(builder, ToXContent.EMPTY_PARAMS); - return new XContentSource(builder); - } - } - /** * Custom plugin that registers XPack script context. */