Tests: Add rolling upgrade tests for watcher (#32428)

These tests ensure, that the basic watch APIs are tested in the rolling
upgrade tests. After initially adding a watch, the tests try to get,
execute, deactivate and activate a watch. Watcher stats are tested as
well, and an own java based test has been added for restarting, as that
requires waiting for a state change. Watcher history is also checked.

Closes #31216

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.upgrades;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+public class WatcherRestartIT extends AbstractUpgradeTestCase {
+
+    public void testWatcherRestart() throws Exception {
+        client().performRequest(new Request("POST", "/_xpack/watcher/_stop"));
+        ensureWatcherStopped();
+
+        client().performRequest(new Request("POST", "/_xpack/watcher/_start"));
+        ensureWatcherStarted();
+    }
+
+    private void ensureWatcherStopped() throws Exception {
+        assertBusy(() -> {
+            Response stats = client().performRequest(new Request("GET", "_xpack/watcher/stats"));
+            String responseBody = EntityUtils.toString(stats.getEntity(), StandardCharsets.UTF_8);
+            assertThat(responseBody, containsString("\"watcher_state\":\"stopped\""));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"starting\"")));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"started\"")));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"stopping\"")));
+        });
+    }
+
+    private void ensureWatcherStarted() throws Exception {
+        assertBusy(() -> {
+            Response response = client().performRequest(new Request("GET", "_xpack/watcher/stats"));
+            String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+            assertThat(responseBody, containsString("\"watcher_state\":\"started\""));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"starting\"")));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"stopping\"")));
+            assertThat(responseBody, not(containsString("\"watcher_state\":\"stopped\"")));
+        });
+    }
+}

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/60_watcher.yml

@@ -0,0 +1,74 @@
+---
+"CRUD watch APIs":
+
+  # no need to put watch, exists already
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+
+
+  # execute watch
+  - do:
+      xpack.watcher.execute_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "record_execution" : true
+          }
+  - set: { _id : record_id }
+  - match: { watch_record.watch_id: "my_watch" }
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.status.state.active: true }
+
+  # check watch history entry
+  - do:
+      indices.refresh:
+        index: .watcher-history-*
+
+  - do:
+      search:
+        index: .watcher-history-*
+        body:
+          {
+            "query" : { "term" : { "_id" : "$record_id" } }
+          }
+  - match: { hits.total: 1 }
+
+  # deactivate watch, check with GET API as well
+  - do:
+      xpack.watcher.deactivate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : false }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: false }
+
+
+  # activate watch again, check with GET API as well
+  - do:
+      xpack.watcher.activate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : true }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: true }
+
+
+---
+"Test watcher stats output":
+  - do:
+      xpack.watcher.stats: {}
+  - match: { "manually_stopped": false }
+  - match: { "stats.0.watcher_state": "started" }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/60_watcher.yml

@@ -0,0 +1,94 @@
+---
+"CRUD watch APIs":
+
+  - do:
+      xpack.watcher.put_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "trigger": {
+              "schedule" : { "cron" : "0 0 0 1 * ? 2099" }
+            },
+            "input": {
+              "simple": {}
+            },
+            "actions": {
+              "logging": {
+                "logging": {
+                  "text": "my logging action"
+                }
+              }
+            }
+          }
+  - match: { _id: "my_watch" }
+
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+
+
+  # execute watch
+  - do:
+      xpack.watcher.execute_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "record_execution" : true
+          }
+  - set: { _id : record_id }
+  - match: { watch_record.watch_id: "my_watch" }
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.status.state.active: true }
+
+  # check watch history entry
+  - do:
+      indices.refresh:
+        index: .watcher-history-*
+
+  - do:
+      search:
+        index: .watcher-history-*
+        body:
+          {
+            "query" : { "term" : { "_id" : "$record_id" } }
+          }
+  - match: { hits.total: 1 }
+
+  # deactivate watch, check with GET API as well
+  - do:
+      xpack.watcher.deactivate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : false }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: false }
+
+
+  # activate watch again, check with GET API as well
+  - do:
+      xpack.watcher.activate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : true }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: true }
+
+
+---
+"Test watcher stats output":
+  - do:
+      xpack.watcher.stats: {}
+  - match: { "manually_stopped": false }
+  - match: { "stats.0.watcher_state": "started" }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/60_watcher.yml

@@ -0,0 +1,73 @@
+---
+"CRUD watch APIs":
+
+  # no need to put watch, exists already
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+
+
+  # execute watch
+  - do:
+      xpack.watcher.execute_watch:
+        id: "my_watch"
+        body:  >
+          {
+            "record_execution" : true
+          }
+  - set: { _id : record_id }
+  - match: { watch_record.watch_id: "my_watch" }
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.status.state.active: true }
+
+  # check watch history entry
+  - do:
+      indices.refresh:
+        index: .watcher-history-*
+
+  - do:
+      search:
+        index: .watcher-history-*
+        body:
+          {
+            "query" : { "term" : { "_id" : "$record_id" } }
+          }
+  - match: { hits.total: 1 }
+
+  # deactivate watch, check with GET API as well
+  - do:
+      xpack.watcher.deactivate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : false }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: false }
+
+
+  # activate watch again, check with GET API as well
+  - do:
+      xpack.watcher.activate_watch:
+        watch_id: "my_watch"
+  - match: { status.state.active : true }
+  
+  - do:
+      xpack.watcher.get_watch:
+        id: "my_watch"
+  - match: { found : true}
+  - match: { _id: "my_watch" }
+  - match: { status.state.active: true }
+
+---
+"Test watcher stats output":
+  - do:
+      xpack.watcher.stats: {}
+  - match: { "manually_stopped": false }
+  - match: { "stats.0.watcher_state": "started" }