[7.x] Refactor watcher tests (#52799) (#52844)

This PR moves the majority of the Watcher REST tests under
the Watcher x-pack plugin.

Specifically, moves the Watcher tests from:
x-pack/plugin/test
x-pack/qa/smoke-test-watcher
x-pack/qa/smoke-test-watcher-with-security
x-pack/qa/smoke-test-monitoring-with-watcher

to:
x-pack/plugin/watcher/qa/rest (/test and /qa/smoke-test-watcher)
x-pack/plugin/watcher/qa/with-security
x-pack/plugin/watcher/qa/with-monitoring

Additionally, this disables Watcher from the main
x-pack test cluster and consolidates the stop/start logic
for the tests listed.

No changes to the tests (beyond moving them) are included.

3rd party tests and doc tests (which also touch Watcher)
are not included in the changes here.

x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java

@@ -103,7 +103,6 @@ public class XDocsClientYamlTestSuiteIT extends XPackRestIT {
         }
     }
 
-    @Override
     protected boolean isWatcherTest() {
         String testName = getTestName();
         return testName != null && (testName.contains("watcher/") || testName.contains("watcher\\"));

x-pack/plugin/build.gradle

@@ -63,7 +63,6 @@ subprojects {
   }
 }
 
-// https://github.com/elastic/x-plugins/issues/724
 configurations {
   testArtifacts.extendsFrom testRuntime
   restXpackSpecs
@@ -141,6 +140,7 @@ testClusters.integTest {
   testDistribution = 'DEFAULT' // this is important since we use the reindex module in ML
   setting 'xpack.ml.enabled', 'true'
   setting 'xpack.security.enabled', 'true'
+  setting 'xpack.watcher.enabled', 'false'
   // Integration tests are supposed to enable/disable exporters before/after each test
   setting 'xpack.monitoring.exporters._local.type', 'local'
   setting 'xpack.monitoring.exporters._local.enabled', 'false'

x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityIT.java

@@ -58,11 +58,6 @@ public class MlWithSecurityIT extends XPackRestIT {
         return false;
     }
 
-    @Override
-    protected boolean isWatcherTest() {
-        return false;
-    }
-
     @Override
     protected boolean isMachineLearningTest() {
         return true;

x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java

@@ -6,11 +6,8 @@
 package org.elasticsearch.xpack.test.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.apache.http.HttpStatus;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -21,14 +18,12 @@ import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.ObjectPath;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
-import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField;
 import org.junit.After;
 import org.junit.Before;
 
@@ -50,7 +45,6 @@ import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_A
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
 
 /** Runs rest tests against external cluster */
 public class XPackRestIT extends ESClientYamlSuiteTestCase {
@@ -77,7 +71,6 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase {
     @Before
     public void setupForTests() throws Exception {
         waitForTemplates();
-        waitForWatcher();
         enableMonitoring();
     }
 
@@ -103,60 +96,7 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase {
         }
     }
 
-    private void waitForWatcher() throws Exception {
-        // ensure watcher is started, so that a test can stop watcher and everything still works fine
-        if (isWatcherTest()) {
-            assertBusy(() -> {
-                ClientYamlTestResponse response =
-                    getAdminExecutionContext().callApi("watcher.stats", emptyMap(), emptyList(), emptyMap());
-                String state = (String) response.evaluate("stats.0.watcher_state");
 
-                switch (state) {
-                    case "stopped":
-                        ClientYamlTestResponse startResponse =
-                            getAdminExecutionContext().callApi("watcher.start", emptyMap(), emptyList(), emptyMap());
-                        boolean isAcknowledged = (boolean) startResponse.evaluate("acknowledged");
-                        assertThat(isAcknowledged, is(true));
-                        throw new AssertionError("waiting until stopped state reached started state");
-                    case "stopping":
-                        throw new AssertionError("waiting until stopping state reached stopped state to start again");
-                    case "starting":
-                        throw new AssertionError("waiting until starting state reached started state");
-                    case "started":
-                        // all good here, we are done
-                        break;
-                    default:
-                        throw new AssertionError("unknown state[" + state + "]");
-                }
-            });
-
-            for (String template : WatcherIndexTemplateRegistryField.TEMPLATE_NAMES) {
-                awaitCallApi("indices.exists_template", singletonMap("name", template), emptyList(),
-                    response -> true,
-                    () -> "Exception when waiting for [" + template + "] template to be created");
-            }
-
-            boolean existsWatcherIndex = adminClient()
-                    .performRequest(new Request("HEAD", ".watches"))
-                    .getStatusLine().getStatusCode() == 200;
-            if (existsWatcherIndex == false) {
-                return;
-            }
-            Request searchWatchesRequest = new Request("GET", ".watches/_search");
-            searchWatchesRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
-            searchWatchesRequest.addParameter("size", "1000");
-            Response response = adminClient().performRequest(searchWatchesRequest);
-            ObjectPath objectPathResponse = ObjectPath.createFromResponse(response);
-            int totalHits = objectPathResponse.evaluate("hits.total");
-            if (totalHits > 0) {
-                List<Map<String, Object>> hits = objectPathResponse.evaluate("hits.hits");
-                for (Map<String, Object> hit : hits) {
-                    String id = (String) hit.get("_id");
-                    adminClient().performRequest(new Request("DELETE", "_watcher/watch/" + id));
-                }
-            }
-        }
-    }
 
     /**
      * Enable monitoring and waits for monitoring documents to be collected and indexed in
@@ -314,11 +254,6 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase {
         return testName != null && (testName.contains("=monitoring/") || testName.contains("=monitoring\\"));
     }
 
-    protected boolean isWatcherTest() {
-        String testName = getTestName();
-        return testName != null && (testName.contains("=watcher/") || testName.contains("=watcher\\"));
-    }
-
     protected boolean isMachineLearningTest() {
         String testName = getTestName();
         return testName != null && (testName.contains("=ml/") || testName.contains("=ml\\"));

x-pack/plugin/watcher/build.gradle

@@ -139,3 +139,12 @@ test {
 // installing them as individual plugins for integ tests doesn't make sense,
 // so we disable integ tests
 integTest.enabled = false
+
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+  project.subprojects
+    .find { it.path == project.path + ":qa" }
+    .subprojects
+    .findAll { it.path.startsWith(project.path + ":qa") }
+    .each { check.dependsOn it.check }
+}

x-pack/plugin/watcher/qa/rest/build.gradle

@@ -6,9 +6,22 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
+configurations {
+  testArtifacts.extendsFrom testRuntime
+}
+
+task testJar(type: Jar) {
+  appendix 'test'
+  from sourceSets.test.output
+}
+
+artifacts {
+  testArtifacts testJar
+}
+
 restResources {
   restApi {
-    includeXpack 'watcher'
+    includeXpack 'watcher', 'xpack'
   }
 }
 

x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java

@@ -14,15 +14,11 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
-import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
-import org.junit.After;
-import org.junit.Before;
+import org.elasticsearch.xpack.watcher.WatcherRestTestCase;
 
 import java.io.IOException;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -33,71 +29,11 @@ import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SmokeTestWatcherTestSuiteIT extends ESRestTestCase {
+public class SmokeTestWatcherTestSuiteIT extends WatcherRestTestCase {
 
     private static final String TEST_ADMIN_USERNAME = "test_admin";
     private static final String TEST_ADMIN_PASSWORD = "x-pack-test-password";
 
-    @Before
-    public void startWatcher() throws Exception {
-        // delete the watcher history to not clutter with entries from other test
-        assertOK(adminClient().performRequest(new Request("DELETE", "/.watcher-history-*")));
-
-        assertBusy(() -> {
-            Response response = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
-            String state = ObjectPath.createFromResponse(response).evaluate("stats.0.watcher_state");
-
-            switch (state) {
-                case "stopped":
-                    Response startResponse = adminClient().performRequest(new Request("POST", "/_watcher/_start"));
-                    boolean isAcknowledged = ObjectPath.createFromResponse(startResponse).evaluate("acknowledged");
-                    assertThat(isAcknowledged, is(true));
-                    throw new AssertionError("waiting until stopped state reached started state");
-                case "stopping":
-                    throw new AssertionError("waiting until stopping state reached stopped state to start again");
-                case "starting":
-                    throw new AssertionError("waiting until starting state reached started state");
-                case "started":
-                    // all good here, we are done
-                    break;
-                default:
-                    throw new AssertionError("unknown state[" + state + "]");
-            }
-        });
-
-        assertBusy(() -> {
-            for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) {
-                Response templateExistsResponse = adminClient().performRequest(new Request("HEAD", "/_template/" + template));
-                assertThat(templateExistsResponse.getStatusLine().getStatusCode(), is(200));
-            }
-        });
-    }
-
-    @After
-    public void stopWatcher() throws Exception {
-        assertBusy(() -> {
-            Response response = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
-            String state = ObjectPath.createFromResponse(response).evaluate("stats.0.watcher_state");
-
-            switch (state) {
-                case "stopped":
-                    // all good here, we are done
-                    break;
-                case "stopping":
-                    throw new AssertionError("waiting until stopping state reached stopped state");
-                case "starting":
-                    throw new AssertionError("waiting until starting state reached started state to stop");
-                case "started":
-                    Response stopResponse = adminClient().performRequest(new Request("POST", "/_watcher/_stop"));
-                    boolean isAcknowledged = ObjectPath.createFromResponse(stopResponse).evaluate("acknowledged");
-                    assertThat(isAcknowledged, is(true));
-                    throw new AssertionError("waiting until started state reached stopped state");
-                default:
-                    throw new AssertionError("unknown state[" + state + "]");
-            }
-        }, 60, TimeUnit.SECONDS);
-    }
-
     @Override
     protected Settings restClientSettings() {
         String token = basicAuthHeaderValue("watcher_manager", new SecureString("x-pack-test-password".toCharArray()));

x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/smoketest/WatcherYamlRestIT.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.smoketest;
+
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.xpack.watcher.WatcherYamlSuiteTestCase;
+
+/**
+ * Runs the YAML rest tests against an external cluster
+ */
+public class WatcherYamlRestIT extends WatcherYamlSuiteTestCase {
+    public WatcherYamlRestIT(ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+}

x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherRestTestCase.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Parent test class for Watcher (not-YAML) based REST tests
+ */
+public abstract class WatcherRestTestCase extends ESRestTestCase {
+
+    @Before
+    public final void startWatcher() throws Exception {
+        assertBusy(() -> {
+            Response response = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
+            String state = ObjectPath.createFromResponse(response).evaluate("stats.0.watcher_state");
+
+            switch (state) {
+                case "stopped":
+                    Response startResponse = adminClient().performRequest(new Request("POST", "/_watcher/_start"));
+                    boolean isAcknowledged = ObjectPath.createFromResponse(startResponse).evaluate("acknowledged");
+                    assertThat(isAcknowledged, is(true));
+                    throw new AssertionError("waiting until stopped state reached started state");
+                case "stopping":
+                    throw new AssertionError("waiting until stopping state reached stopped state to start again");
+                case "starting":
+                    throw new AssertionError("waiting until starting state reached started state");
+                case "started":
+                    // all good here, we are done
+                    break;
+                default:
+                    throw new AssertionError("unknown state[" + state + "]");
+            }
+        });
+    }
+
+    @After
+    public final void stopWatcher() throws Exception {
+        assertBusy(() -> {
+            Response response = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
+            String state = ObjectPath.createFromResponse(response).evaluate("stats.0.watcher_state");
+
+            switch (state) {
+                case "stopped":
+                    // all good here, we are done
+                    break;
+                case "stopping":
+                    throw new AssertionError("waiting until stopping state reached stopped state");
+                case "starting":
+                    throw new AssertionError("waiting until starting state reached started state to stop");
+                case "started":
+                    Response stopResponse = adminClient().performRequest(new Request("POST", "/_watcher/_stop"));
+                    boolean isAcknowledged = ObjectPath.createFromResponse(stopResponse).evaluate("acknowledged");
+                    assertThat(isAcknowledged, is(true));
+                    throw new AssertionError("waiting until started state reached stopped state");
+                default:
+                    throw new AssertionError("unknown state[" + state + "]");
+            }
+        }, 60, TimeUnit.SECONDS);
+
+        Request deleteWatchesIndexRequest = new Request("DELETE", ".watches");
+        deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
+        adminClient().performRequest(deleteWatchesIndexRequest);
+
+        Request deleteWatchHistoryRequest = new Request("DELETE", ".watcher-history-*");
+        deleteWatchHistoryRequest.addParameter("ignore_unavailable", "true");
+        adminClient().performRequest(deleteWatchHistoryRequest);
+    }
+}

x-pack/plugin/watcher/qa/rest/src/test/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.smoketest;
+package org.elasticsearch.xpack.watcher;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@@ -11,7 +11,6 @@ import org.elasticsearch.client.Request;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
 import org.junit.After;
 import org.junit.Before;
 
@@ -19,14 +18,14 @@ import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
-/** Runs rest tests against external cluster */
-public class WatcherRestIT extends ESClientYamlSuiteTestCase {
-
-    public WatcherRestIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+/**
+ * Parent test class for Watcher YAML based REST tests
+ */
+public abstract class WatcherYamlSuiteTestCase extends ESClientYamlSuiteTestCase {
+    public WatcherYamlSuiteTestCase(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
@@ -36,7 +35,7 @@ public class WatcherRestIT extends ESClientYamlSuiteTestCase {
     }
 
     @Before
-    public void startWatcher() throws Exception {
+    public final void startWatcher() throws Exception {
         assertBusy(() -> {
             ClientYamlTestResponse response =
                 getAdminExecutionContext().callApi("watcher.stats", emptyMap(), emptyList(), emptyMap());
@@ -60,27 +59,15 @@ public class WatcherRestIT extends ESClientYamlSuiteTestCase {
                     throw new AssertionError("unknown state[" + state + "]");
             }
         });
+    }
 
-        assertBusy(() -> {
-            for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) {
-                ClientYamlTestResponse templateExistsResponse = getAdminExecutionContext().callApi("indices.exists_template",
-                    singletonMap("name", template), emptyList(), emptyMap());
-                assertThat(templateExistsResponse.getStatusCode(), is(200));
-            }
-        });
-    }
 
     @After
-    public void stopWatcher() throws Exception {
-        Request deleteWatchesIndexRequest = new Request("DELETE", "/.watches");
-        deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
-        adminClient().performRequest(deleteWatchesIndexRequest);
-
+    public final void stopWatcher() throws Exception {
         assertBusy(() -> {
             ClientYamlTestResponse response =
                 getAdminExecutionContext().callApi("watcher.stats", emptyMap(), emptyList(), emptyMap());
             String state = (String) response.evaluate("stats.0.watcher_state");
-
             switch (state) {
                 case "stopped":
                     int watcherCount = (int) response.evaluate("stats.0.watch_count");
@@ -101,5 +88,15 @@ public class WatcherRestIT extends ESClientYamlSuiteTestCase {
                     throw new AssertionError("unknown state[" + state + "]");
             }
         }, 60, TimeUnit.SECONDS);
+
+        Request deleteWatchesIndexRequest = new Request("DELETE", ".watches");
+        deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
+        adminClient().performRequest(deleteWatchesIndexRequest);
+
+        Request deleteWatchHistoryRequest = new Request("DELETE", ".watcher-history-*");
+        deleteWatchHistoryRequest.addParameter("ignore_unavailable", "true");
+        adminClient().performRequest(deleteWatchHistoryRequest);
     }
 }
+
+

x-pack/plugin/watcher/qa/with-security/build.gradle

@@ -4,17 +4,24 @@ apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
   testCompile project(':x-pack:qa')
+  testCompile project(path: ':x-pack:plugin:watcher:qa:rest', configuration: 'testArtifacts')
+}
+
+
+// bring in watcher rest test suite from the rest project
+task copyWatcherRestTests(type: Copy) {
+  into project.sourceSets.test.output.resourcesDir
+  from project(xpackProject('plugin:watcher:qa:rest').path).sourceSets.test.resources.srcDirs
+  include 'rest-api-spec/test/watcher/**'
 }
 
 restResources {
   restApi {
     includeXpack 'watcher', 'security', 'xpack'
   }
-  restTests {
-    includeXpack 'watcher'
-  }
 }
 
+integTest.runner.dependsOn copyWatcherRestTests
 testClusters.integTest {
   testDistribution = 'DEFAULT'
   setting 'xpack.ilm.enabled', 'false'

x-pack/plugin/watcher/qa/with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.smoketest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.elasticsearch.xpack.watcher.WatcherYamlSuiteTestCase;
+import org.junit.Before;
+
+import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue;
+
+public class SmokeTestWatcherWithSecurityClientYamlTestSuiteIT extends WatcherYamlSuiteTestCase {
+
+    private static final String TEST_ADMIN_USERNAME = "test_admin";
+    private static final String TEST_ADMIN_PASSWORD = "x-pack-test-password";
+
+    public SmokeTestWatcherWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+    @Before
+    public void beforeTest() throws Exception {
+        // create one document in this index, so we can test in the YAML tests, that the index cannot be accessed
+        Request request = new Request("PUT", "/index_not_allowed_to_read/_doc/1");
+        request.setJsonEntity("{\"foo\":\"bar\"}");
+        adminClient().performRequest(request);
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("watcher_manager", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+                .put(ThreadContext.PREFIX + ".Authorization", token)
+                .build();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecureString(TEST_ADMIN_PASSWORD.toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+}

x-pack/plugin/watcher/qa/with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

@@ -14,26 +14,22 @@ import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
-import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
-import org.junit.After;
+import org.elasticsearch.xpack.watcher.WatcherRestTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
 import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 
-public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
+public class SmokeTestWatcherWithSecurityIT extends WatcherRestTestCase {
 
     private static final String TEST_ADMIN_USERNAME = "test_admin";
     private static final String TEST_ADMIN_PASSWORD = "x-pack-test-password";
@@ -41,85 +37,20 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
     private String watchId = randomAlphaOfLength(20);
 
     @Before
-    public void startWatcher() throws Exception {
+    public void beforeTest() throws Exception {
+        Request deleteRequest = new Request("DELETE", "/my_test_index");
+        deleteRequest.addParameter("ignore_unavailable", "true");
+        adminClient().performRequest(deleteRequest);
+
         Request createAllowedDoc = new Request("PUT", "/my_test_index/_doc/1");
         createAllowedDoc.setJsonEntity("{ \"value\" : \"15\" }");
         createAllowedDoc.addParameter("refresh", "true");
         adminClient().performRequest(createAllowedDoc);
 
-        // delete the watcher history to not clutter with entries from other test
-        adminClient().performRequest(new Request("DELETE", ".watcher-history-*"));
-
-        // create one document in this index, so we can test in the YAML tests, that the index cannot be accessed
+         // create one document in this index, so we can test that the index cannot be accessed
         Request createNotAllowedDoc = new Request("PUT", "/index_not_allowed_to_read/_doc/1");
         createNotAllowedDoc.setJsonEntity("{\"foo\":\"bar\"}");
         adminClient().performRequest(createNotAllowedDoc);
-
-        assertBusy(() -> {
-            try {
-                Response statsResponse = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
-                ObjectPath objectPath = ObjectPath.createFromResponse(statsResponse);
-                String state = objectPath.evaluate("stats.0.watcher_state");
-
-                switch (state) {
-                    case "stopped":
-                        Response startResponse = adminClient().performRequest(new Request("POST", "/_watcher/_start"));
-                        Map<String, Object> responseMap = entityAsMap(startResponse);
-                        assertThat(responseMap, hasEntry("acknowledged", true));
-                        throw new AssertionError("waiting until stopped state reached started state");
-                    case "stopping":
-                        throw new AssertionError("waiting until stopping state reached stopped state to start again");
-                    case "starting":
-                        throw new AssertionError("waiting until starting state reached started state");
-                    case "started":
-                        // all good here, we are done
-                        break;
-                    default:
-                        throw new AssertionError("unknown state[" + state + "]");
-                }
-            } catch (IOException e) {
-                throw new AssertionError(e);
-            }
-        });
-
-        assertBusy(() -> {
-            for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) {
-                assertOK(adminClient().performRequest(new Request("HEAD", "_template/" + template)));
-            }
-        });
-    }
-
-    @After
-    public void stopWatcher() throws Exception {
-
-        assertBusy(() -> {
-            try {
-                Response statsResponse = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
-                ObjectPath objectPath = ObjectPath.createFromResponse(statsResponse);
-                String state = objectPath.evaluate("stats.0.watcher_state");
-
-                switch (state) {
-                case "stopped":
-                    // all good here, we are done
-                    break;
-                case "stopping":
-                    throw new AssertionError("waiting until stopping state reached stopped state");
-                case "starting":
-                    throw new AssertionError("waiting until starting state reached started state to stop");
-                case "started":
-                    Response stopResponse = adminClient().performRequest(new Request("POST", "/_watcher/_stop"));
-                    String body = EntityUtils.toString(stopResponse.getEntity());
-                    assertThat(body, containsString("\"acknowledged\":true"));
-                    throw new AssertionError("waiting until started state reached stopped state");
-                default:
-                    throw new AssertionError("unknown state[" + state + "]");
-                }
-            } catch (IOException e) {
-                throw new AssertionError(e);
-            }
-        }, 60, TimeUnit.SECONDS);
-
-        adminClient().performRequest(new Request("DELETE", "/my_test_index"));
     }
 
     @Override

x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java

@@ -1,130 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.smoketest;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
-import org.junit.After;
-import org.junit.Before;
-
-import java.util.Collections;
-import java.util.concurrent.TimeUnit;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
-import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue;
-import static org.hamcrest.Matchers.is;
-
-public class SmokeTestWatcherWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
-
-    private static final String TEST_ADMIN_USERNAME = "test_admin";
-    private static final String TEST_ADMIN_PASSWORD = "x-pack-test-password";
-
-    public SmokeTestWatcherWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws Exception {
-        return ESClientYamlSuiteTestCase.createParameters();
-    }
-
-    @Before
-    public void startWatcher() throws Exception {
-        // delete the watcher history to not clutter with entries from other test
-        getAdminExecutionContext().callApi("indices.delete", Collections.singletonMap("index", ".watcher-history-*"),
-                emptyList(), emptyMap());
-
-        // create one document in this index, so we can test in the YAML tests, that the index cannot be accessed
-        Request request = new Request("PUT", "/index_not_allowed_to_read/_doc/1");
-        request.setJsonEntity("{\"foo\":\"bar\"}");
-        adminClient().performRequest(request);
-
-        assertBusy(() -> {
-            ClientYamlTestResponse response =
-                    getAdminExecutionContext().callApi("watcher.stats", emptyMap(), emptyList(), emptyMap());
-            String state = (String) response.evaluate("stats.0.watcher_state");
-
-            switch (state) {
-                case "stopped":
-                    ClientYamlTestResponse startResponse =
-                            getAdminExecutionContext().callApi("watcher.start", emptyMap(), emptyList(), emptyMap());
-                    boolean isAcknowledged = (boolean) startResponse.evaluate("acknowledged");
-                    assertThat(isAcknowledged, is(true));
-                    throw new AssertionError("waiting until stopped state reached started state");
-                case "stopping":
-                    throw new AssertionError("waiting until stopping state reached stopped state to start again");
-                case "starting":
-                    throw new AssertionError("waiting until starting state reached started state");
-                case "started":
-                    // all good here, we are done
-                    break;
-                default:
-                    throw new AssertionError("unknown state[" + state + "]");
-            }
-        });
-
-        assertBusy(() -> {
-            for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) {
-                ClientYamlTestResponse templateExistsResponse = getAdminExecutionContext().callApi("indices.exists_template",
-                        singletonMap("name", template), emptyList(), emptyMap());
-                assertThat(templateExistsResponse.getStatusCode(), is(200));
-            }
-        });
-    }
-
-    @After
-    public void stopWatcher() throws Exception {
-        assertBusy(() -> {
-            ClientYamlTestResponse response =
-                    getAdminExecutionContext().callApi("watcher.stats", emptyMap(), emptyList(), emptyMap());
-            String state = (String) response.evaluate("stats.0.watcher_state");
-
-            switch (state) {
-                case "stopped":
-                    // all good here, we are done
-                    break;
-                case "stopping":
-                    throw new AssertionError("waiting until stopping state reached stopped state");
-                case "starting":
-                    throw new AssertionError("waiting until starting state reached started state to stop");
-                case "started":
-                    ClientYamlTestResponse stopResponse =
-                            getAdminExecutionContext().callApi("watcher.stop", emptyMap(), emptyList(), emptyMap());
-                    boolean isAcknowledged = (boolean) stopResponse.evaluate("acknowledged");
-                    assertThat(isAcknowledged, is(true));
-                    throw new AssertionError("waiting until started state reached stopped state");
-                default:
-                    throw new AssertionError("unknown state[" + state + "]");
-            }
-        }, 60, TimeUnit.SECONDS);
-    }
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = basicAuthHeaderValue("watcher_manager", new SecureString("x-pack-test-password".toCharArray()));
-        return Settings.builder()
-                .put(ThreadContext.PREFIX + ".Authorization", token)
-                .build();
-    }
-
-    @Override
-    protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecureString(TEST_ADMIN_PASSWORD.toCharArray()));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            .build();
-    }
-}