Watcher: Fix watch history template for failed inputs (elastic/elasticsearch#2928)

If the result of a search actions fails (i.e. because the index you queried does not exist yet),
the watch record failed to store into the Watch History because of a mapping issue, as the
template path match regular expression did not match properly.

Closes elastic/elasticsearch#2913

Original commit: elastic/x-pack-elasticsearch@3c2d4b3ca9

elasticsearch/x-pack/watcher/src/main/resources/watch_history.json

@@ -12,7 +12,7 @@
       "dynamic_templates": [
         {
           "disabled_payload_fields": {
-            "path_match": "result\\.(input|(transform(\\..+)*)|(actions\\.transform(\\..+)*))\\.payload",
+            "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*))\\.payload",
             "match_pattern": "regex",
             "mapping": {
               "type": "object",
@@ -22,7 +22,7 @@
         },
         {
           "disabled_search_request_body_fields": {
-            "path_match": "result\\.(input(\\..+)|(transform(\\..+)*)|(actions\\.transform(\\..+)*))\\.search\\.request\\.(body|template)",
+            "path_match": "result\\.(input(\\..+)*|(transform(\\..+)*)|(actions\\.transform(\\..+)*))\\.search\\.request\\.(body|template)",
             "match_pattern": "regex",
             "mapping": {
               "type": "object",

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/test/integration/HistoryIntegrationTests.java

@@ -5,22 +5,34 @@
  */
 package org.elasticsearch.watcher.test.integration;
 
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
 import org.elasticsearch.xpack.watcher.client.WatchSourceBuilder;
-import org.elasticsearch.xpack.watcher.client.WatchSourceBuilders;
+import org.elasticsearch.xpack.watcher.input.Input;
-import org.elasticsearch.xpack.watcher.input.InputBuilders;
+import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.transport.actions.put.PutWatchResponse;
-import org.elasticsearch.xpack.watcher.trigger.TriggerBuilders;
+import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
-import org.elasticsearch.xpack.watcher.trigger.schedule.Schedules;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.loggingAction;
+import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
+import static org.elasticsearch.xpack.watcher.input.InputBuilders.chainInput;
+import static org.elasticsearch.xpack.watcher.input.InputBuilders.searchInput;
+import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
+import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.newInputSearchRequest;
+import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
+import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
 import static org.hamcrest.Matchers.is;
 
 public class HistoryIntegrationTests extends AbstractWatcherIntegrationTestCase {
@@ -32,13 +44,13 @@ public class HistoryIntegrationTests extends AbstractWatcherIntegrationTestCase
         index("foo", "bar", "1", xContentBuilder);
         refresh();
 
-        WatchSourceBuilder builder = WatchSourceBuilders.watchBuilder()
+        WatchSourceBuilder builder = watchBuilder()
-                .trigger(TriggerBuilders.schedule(Schedules.interval("10s")))
+                .trigger(schedule(interval("10s")))
-                .addAction("logging", ActionBuilders.loggingAction("foo"));
+                .addAction("logging", loggingAction("foo"));
 
         SearchRequestBuilder searchRequestBuilder = client().prepareSearch("foo").addSort(SortBuilders.fieldSort("inner.date").order
                 (SortOrder.DESC));
-        builder.input(InputBuilders.chainInput().add("first", InputBuilders.searchInput(searchRequestBuilder.request())));
+        builder.input(chainInput().add("first", searchInput(searchRequestBuilder.request())));
 
         PutWatchResponse response = watcherClient().preparePutWatch("test_watch").setSource(builder).get();
         assertThat(response.isCreated(), is(true));
@@ -49,4 +61,89 @@ public class HistoryIntegrationTests extends AbstractWatcherIntegrationTestCase
         SearchResponse searchResponse = client().prepareSearch(".watcher-history-*").get();
         assertHitCount(searchResponse, 1);
     }
+
+    // See https://github.com/elastic/x-plugins/issues/2913
+    public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exception {
+        SearchRequest sortSearchRequest = newInputSearchRequest("non-existing-index")
+                .source(searchSource()
+                        .query(matchAllQuery())
+                        .sort("trigger_event.triggered_time", SortOrder.DESC)
+                        .size(1));
+
+        // The result of the search input will be a failure, because a missing index does not exist when
+        // the query is executed
+        Input.Builder input = searchInput(sortSearchRequest);
+        // wrapping this randomly into a chained input to test this as well
+        boolean useChained = randomBoolean();
+        if (useChained) {
+            input = chainInput().add("chained", input);
+        }
+
+        watcherClient().preparePutWatch("test_watch")
+                .setSource(watchBuilder()
+                        .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.HOURS)))
+                        .input(input)
+                        .addAction("_logger", loggingAction("#### randomLogging")))
+                .get();
+
+        watcherClient().prepareExecuteWatch("test_watch").setRecordExecution(true).get();
+
+        flush(".watcher-history*");
+        SearchResponse searchResponse = client().prepareSearch(".watcher-history*").setSize(0).get();
+        assertHitCount(searchResponse, 1);
+
+        // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition
+        GetMappingsResponse response = client().admin().indices().prepareGetMappings(".watcher-history*").addTypes("watch_record").get();
+        byte[] bytes = response.getMappings().values().iterator().next().value.get("watch_record").source().uncompressed();
+        XContentSource source = new XContentSource(new BytesArray(bytes), XContentType.JSON);
+        // lets make sure the body fields are disabled
+        if (useChained) {
+            String chainedPath = "watch_record.properties.result.properties.input.properties.chain.properties.chained.properties.search" +
+                    ".properties.request.properties.body.enabled";
+            assertThat(source.getValue(chainedPath), is(false));
+        } else {
+            String path =
+                    "watch_record.properties.result.properties.input.properties.search.properties.request.properties.body.enabled";
+            assertThat(source.getValue(path), is(false));
+        }
+    }
+
+    // See https://github.com/elastic/x-plugins/issues/2913
+    public void testPayloadInputWithDotsInFieldNameWorks() throws Exception {
+        Input.Builder input = simpleInput("foo.bar", "bar");
+
+        // wrapping this randomly into a chained input to test this as well
+        boolean useChained = randomBoolean();
+        if (useChained) {
+            input = chainInput().add("chained", input);
+        }
+
+        watcherClient().preparePutWatch("test_watch")
+                .setSource(watchBuilder()
+                        .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.HOURS)))
+                        .input(input)
+                        .addAction("_logger", loggingAction("#### randomLogging")))
+                .get();
+
+        watcherClient().prepareExecuteWatch("test_watch").setRecordExecution(true).get();
+
+        flush(".watcher-history*");
+        SearchResponse searchResponse = client().prepareSearch(".watcher-history*").setSize(0).get();
+        assertHitCount(searchResponse, 1);
+
+        // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition
+        GetMappingsResponse response = client().admin().indices().prepareGetMappings(".watcher-history*").addTypes("watch_record").get();
+        byte[] bytes = response.getMappings().values().iterator().next().value.get("watch_record").source().uncompressed();
+        XContentSource source = new XContentSource(new BytesArray(bytes), XContentType.JSON);
+
+        // lets make sure the body fields are disabled
+        if (useChained) {
+            String path = "watch_record.properties.result.properties.input.properties.chain.properties.chained.properties.payload.enabled";
+            assertThat(source.getValue(path), is(false));
+        } else {
+            String path = "watch_record.properties.result.properties.input.properties.payload.enabled";
+            assertThat(source.getValue(path), is(false));
+        }
+
+    }
 }