Watcher: Include watch status in history (elastic/elasticsearch#4875)

In order to help watcher UI easily gather the status of a watch
and its actions, we should write the watch status in the history
as well.

This commit is doing exactly that. Tests ensure, that the status is not searchable,
as in the watch mapping itself.

This also requires the history template to be changed, thus the counter of the template
had to be increased as well.

Some minor refactorings by making ctors private that dont need to be public
have been happening as well.

Closes elastic/elasticsearch#4735

Original commit: elastic/x-pack-elasticsearch@df8352ceb5

elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/actions/ActionStatus.java

@@ -330,18 +330,14 @@ public class ActionStatus implements ToXContentObject {
 
             Execution execution = (Execution) o;
 
-            if (successful != execution.successful) return false;
+            return Objects.equals(successful, execution.successful) &&
-            if (!timestamp.equals(execution.timestamp)) return false;
+                   Objects.equals(timestamp, execution.timestamp) &&
-            return !(reason != null ? !reason.equals(execution.reason) : execution.reason != null);
+                   Objects.equals(reason, execution.reason);
-
         }
 
         @Override
         public int hashCode() {
-            int result = timestamp.hashCode();
+            return Objects.hash(timestamp, successful, reason);
-            result = 31 * result + (successful ? 1 : 0);
-            result = 31 * result + (reason != null ? reason.hashCode() : 0);
-            return result;
         }
 
         @Override
@@ -436,16 +432,12 @@ public class ActionStatus implements ToXContentObject {
             if (o == null || getClass() != o.getClass()) return false;
 
             Throttle throttle = (Throttle) o;
-
+            return Objects.equals(timestamp, throttle.timestamp) && Objects.equals(reason, throttle.reason);
-            if (!timestamp.equals(throttle.timestamp)) return false;
-            return reason.equals(throttle.reason);
         }
 
         @Override
         public int hashCode() {
-            int result = timestamp.hashCode();
+            return Objects.hash(timestamp, reason);
-            result = 31 * result + reason.hashCode();
-            return result;
         }
 
         @Override

elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/execution/WatchExecutionResult.java

@@ -33,8 +33,9 @@ public class WatchExecutionResult implements ToXContentObject {
                 context.actionsResults());
     }
 
-    WatchExecutionResult(DateTime executionTime, long executionDurationMs, Input.Result inputResult, Condition.Result conditionResult,
+    private WatchExecutionResult(DateTime executionTime, long executionDurationMs, Input.Result inputResult,
-                         @Nullable Transform.Result transformResult, Map<String, ActionWrapper.Result> actionsResults) {
+                                 Condition.Result conditionResult, @Nullable Transform.Result transformResult,
+                                 Map<String, ActionWrapper.Result> actionsResults) {
         this.executionTime = executionTime;
         this.inputResult = inputResult;
         this.conditionResult = conditionResult;

elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/history/WatchRecord.java

@@ -33,6 +33,7 @@ import java.util.Objects;
 public abstract class WatchRecord implements ToXContentObject {
 
     protected final Wid id;
+    protected final Watch watch;
     protected final TriggerEvent triggerEvent;
     protected final ExecutionState state;
 
@@ -44,8 +45,8 @@ public abstract class WatchRecord implements ToXContentObject {
     @Nullable protected final Map<String,Object> metadata;
     @Nullable protected final WatchExecutionResult executionResult;
 
-    public WatchRecord(Wid id, TriggerEvent triggerEvent, ExecutionState state, Map<String, Object> vars, ExecutableInput input,
+    private WatchRecord(Wid id, TriggerEvent triggerEvent, ExecutionState state, Map<String, Object> vars, ExecutableInput input,
-                       Condition condition, Map<String, Object> metadata, WatchExecutionResult executionResult) {
+                       Condition condition, Map<String, Object> metadata, Watch watch, WatchExecutionResult executionResult) {
         this.id = id;
         this.triggerEvent = triggerEvent;
         this.state = state;
@@ -54,24 +55,26 @@ public abstract class WatchRecord implements ToXContentObject {
         this.condition = condition;
         this.metadata = metadata;
         this.executionResult = executionResult;
+        this.watch = watch;
     }
 
-    public WatchRecord(Wid id, TriggerEvent triggerEvent, ExecutionState state) {
+    private WatchRecord(Wid id, TriggerEvent triggerEvent, ExecutionState state) {
-        this(id, triggerEvent, state, Collections.emptyMap(), null, null, null, null);
+        this(id, triggerEvent, state, Collections.emptyMap(), null, null, null, null, null);
     }
 
-    public WatchRecord(WatchRecord record, ExecutionState state) {
+    private WatchRecord(WatchRecord record, ExecutionState state) {
-        this(record.id, record.triggerEvent, state, record.vars, record.input, record.condition, record.metadata, record.executionResult);
+        this(record.id, record.triggerEvent, state, record.vars, record.input, record.condition, record.metadata, record.watch,
+                record.executionResult);
     }
 
-    public WatchRecord(WatchExecutionContext context, ExecutionState state) {
+    private WatchRecord(WatchExecutionContext context, ExecutionState state) {
         this(context.id(), context.triggerEvent(), state, context.vars(), context.watch().input(), context.watch().condition(),
-                context.watch().metadata(), null);
+                context.watch().metadata(), context.watch(), null);
     }
 
-    public WatchRecord(WatchExecutionContext context, WatchExecutionResult executionResult) {
+    private WatchRecord(WatchExecutionContext context, WatchExecutionResult executionResult) {
         this(context.id(), context.triggerEvent(), getState(executionResult), context.vars(), context.watch().input(),
-                context.watch().condition(), context.watch().metadata(), executionResult);
+                context.watch().condition(), context.watch().metadata(), context.watch(), executionResult);
     }
 
     private static ExecutionState getState(WatchExecutionResult executionResult) {
@@ -125,6 +128,10 @@ public abstract class WatchRecord implements ToXContentObject {
         builder.field(Field.WATCH_ID.getPreferredName(), id.watchId());
         builder.field(Field.STATE.getPreferredName(), state.id());
 
+        if (watch != null && watch.status() != null) {
+            builder.field("_status", watch.status(), params);
+        }
+
         builder.field(Field.TRIGGER_EVENT.getPreferredName());
         triggerEvent.recordXContent(builder, params);
 
@@ -179,6 +186,7 @@ public abstract class WatchRecord implements ToXContentObject {
         ParseField TRIGGER_EVENT = new ParseField("trigger_event");
         ParseField MESSAGES = new ParseField("messages");
         ParseField STATE = new ParseField("state");
+        ParseField STATUS = new ParseField("_status");
         ParseField VARS = new ParseField("vars");
         ParseField METADATA = new ParseField("metadata");
         ParseField EXECUTION_RESULT = new ParseField("result");

elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java

@@ -37,7 +37,7 @@ import static java.util.Collections.unmodifiableMap;
 public class WatcherIndexTemplateRegistry extends AbstractComponent implements ClusterStateListener {
 
     private static final String FORBIDDEN_INDEX_SETTING = "index.mapper.dynamic";
-    public static final String INDEX_TEMPLATE_VERSION = "2";
+    public static final String INDEX_TEMPLATE_VERSION = "3";
 
     public static final String HISTORY_TEMPLATE_NAME = "watch_history_" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = "triggered_watches";

elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java

@@ -90,6 +90,8 @@ public class TransportExecuteWatchAction extends WatcherTransportAction<ExecuteW
                 // should be executed async in the future
                 GetResponse getResponse = client.getWatch(request.getId());
                 Watch watch = watchParser.parse(request.getId(), true, getResponse.getSourceAsBytesRef(), XContentType.JSON);
+                watch.version(getResponse.getVersion());
+                watch.status().version(getResponse.getVersion());
                 ExecuteWatchResponse executeWatchResponse = executeWatch(request, watch, true);
                 listener.onResponse(executeWatchResponse);
             } catch (IOException e) {

elasticsearch/src/main/resources/watch_history.json

@@ -95,6 +95,11 @@
         "state": {
           "type": "keyword"
         },
+        "_status": {
+          "type": "object",
+          "enabled" : false,
+          "dynamic" : true
+        },
         "messages": {
           "type": "text"
         },

elasticsearch/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java

@@ -8,7 +8,6 @@ package org.elasticsearch.xpack.watcher.support;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;

elasticsearch/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java

@@ -10,8 +10,10 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.xpack.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.watcher.input.Input;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
@@ -19,6 +21,9 @@ import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.transport.actions.put.PutWatchResponse;
 import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
+import org.elasticsearch.xpack.watcher.watch.WatchStatus;
+
+import java.util.Locale;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -33,6 +38,7 @@ import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.templateRequ
 import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
 import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 
 public class HistoryIntegrationTests extends AbstractWatcherIntegrationTestCase {
 
@@ -142,6 +148,53 @@ public class HistoryIntegrationTests extends AbstractWatcherIntegrationTestCase
             String path = "watch_record.properties.result.properties.input.properties.payload.enabled";
             assertThat(source.getValue(path), is(false));
         }
+    }
 
+    public void testThatHistoryContainsStatus() throws Exception {
+        watcherClient().preparePutWatch("test_watch")
+                .setSource(watchBuilder()
+                        .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.HOURS)))
+                        .input(simpleInput("foo", "bar"))
+                        .addAction("_logger", loggingAction("#### randomLogging")))
+                .get();
+
+        watcherClient().prepareExecuteWatch("test_watch").setRecordExecution(true).get();
+
+        WatchStatus status = watcherClient().prepareGetWatch("test_watch").get().getStatus();
+
+        refresh(".watcher-history*");
+        SearchResponse searchResponse = client().prepareSearch(".watcher-history*").setSize(1).get();
+        assertHitCount(searchResponse, 1);
+        SearchHit hit = searchResponse.getHits().getAt(0);
+
+        XContentSource source = new XContentSource(hit.getSourceRef(), XContentType.JSON);
+
+        Boolean active = source.getValue("_status.state.active");
+        assertThat(active, is(status.state().isActive()));
+
+        String timestamp = source.getValue("_status.state.timestamp");
+        assertThat(timestamp, is(status.state().getTimestamp().toString()));
+
+        String lastChecked = source.getValue("_status.last_checked");
+        assertThat(lastChecked, is(status.lastChecked().toString()));
+
+        Integer version = source.getValue("_status.version");
+        int expectedVersion = (int) (status.version() - 1);
+        assertThat(version, is(expectedVersion));
+
+        ActionStatus actionStatus = status.actionStatus("_logger");
+        String ackStatusState = source.getValue("_status.actions._logger.ack.state").toString().toUpperCase(Locale.ROOT);
+        assertThat(ackStatusState, is(actionStatus.ackStatus().state().toString()));
+
+        Boolean lastExecutionSuccesful = source.getValue("_status.actions._logger.last_execution.successful");
+        assertThat(lastExecutionSuccesful, is(actionStatus.lastExecution().successful()));
+
+        // also ensure that the _status field is disabled in the watch history
+        GetMappingsResponse response = client().admin().indices().prepareGetMappings(".watcher-history*").addTypes("watch_record").get();
+        byte[] bytes = response.getMappings().values().iterator().next().value.get("watch_record").source().uncompressed();
+        XContentSource mappingSource = new XContentSource(new BytesArray(bytes), XContentType.JSON);
+        assertThat(mappingSource.getValue("watch_record.properties._status.enabled"), is(false));
+        assertThat(mappingSource.getValue("watch_record.properties._status.properties.status"), is(nullValue()));
+        assertThat(mappingSource.getValue("watch_record.properties._status.properties.status.properties.active"), is(nullValue()));
     }
 }

elasticsearch/src/test/resources/rest-api-spec/test/watcher/execute_watch/10_basic.yaml

@@ -56,6 +56,7 @@ teardown:
   - match: { watch_record.trigger_event.triggered_time: "2012-12-12T12:12:12.120Z" }
   - match: { watch_record.trigger_event.manual.schedule.scheduled_time: "2000-12-12T12:12:12.120Z" }
   - match: { watch_record.state: "executed" }
+  - match: { watch_record._status.state.active: true }
 
 ---
 "Execute unknown watch results in 404":