7x watcher backport testfixes (#43848)

* fix org.elasticsearch.xpack.watcher.test.integration.RejectedExecutionTests (#41777)

This commit un-mutes org.elasticsearch.xpack.watcher.test.integration.RejectedExecutionTests
which was failing intermittently due to a logic bug. It is not possible to use the real
Watcher scheduler (which is needed for this test) and reliabliby count the .triggered-watches
since current count of documents in the .triggered-watches index is based on the timing of the
scheduler and the ability to delete based on the Watcher and Write thread pools.

This commit simply removes the .triggered-watch check and relies soley on the .watcher-history
index as an indication that operations that can occur when the Watcher threadpool is rejecting.

closes #41734

* fix unlikely bug that can prevent Watcher from restarting (#42030)

The bug fixed here is unlikely to happen. It requires ES to be started with
ILM disabled, Watcher enabled, and Watcher explicitly stopped and restarted.
Due to template validation Watcher does not fully start and can result in a
partially started state. This is an unlikely scenerio outside of the testing
framework.

Note - this bug was introduced while the test that would have caught it was
muted. The test remains muted since the underlying cuase of the random failures
has not been identified. When this test is un-muted it will now work.

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java

@@ -79,7 +79,8 @@ public class WatcherIndexTemplateRegistry extends IndexTemplateRegistry {
     }
 
     public static boolean validate(ClusterState state) {
-        return state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) &&
+        return (state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) ||
+            state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) &&
             state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) &&
             state.getMetaData().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME);
     }

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java

@@ -15,8 +15,6 @@ import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateReque
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
 
-import java.util.concurrent.TimeUnit;
-
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
 import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.loggingAction;
@@ -25,7 +23,6 @@ import static org.elasticsearch.xpack.watcher.input.InputBuilders.searchInput;
 import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.templateRequest;
 import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
 import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class RejectedExecutionTests extends AbstractWatcherIntegrationTestCase {
@@ -36,8 +33,7 @@ public class RejectedExecutionTests extends AbstractWatcherIntegrationTestCase {
         return false;
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/41734")
-    public void testHistoryAndTriggeredOnRejection() throws Exception {
+    public void testHistoryOnRejection() throws Exception {
         WatcherClient watcherClient = watcherClient();
         createIndex("idx");
         client().prepareIndex("idx", "_doc").setSource("field", "a").get();
@@ -56,11 +52,7 @@ public class RejectedExecutionTests extends AbstractWatcherIntegrationTestCase {
             flushAndRefresh(".watcher-history-*");
             SearchResponse searchResponse = client().prepareSearch(".watcher-history-*").get();
             assertThat(searchResponse.getHits().getTotalHits().value, greaterThanOrEqualTo(2L));
-        }, 10, TimeUnit.SECONDS);
-
-        flushAndRefresh(".triggered_watches");
-        SearchResponse searchResponse = client().prepareSearch(".triggered_watches").get();
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
+        });
     }
 
     @Override

x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

@@ -83,7 +83,7 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
         });
 
         assertBusy(() -> {
-            for (String template : XPackRestTestConstants.TEMPLATE_NAMES) {
+            for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) {
                 assertOK(adminClient().performRequest(new Request("HEAD", "_template/" + template)));
             }
         });

x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java

@@ -13,13 +13,10 @@ public final class XPackRestTestConstants {
 
     // Watcher constants:
     public static final String INDEX_TEMPLATE_VERSION = "9";
-    public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
     public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";
     public static final String WATCHES_TEMPLATE_NAME = ".watches";
-    public static final String[] TEMPLATE_NAMES = new String[] {
-        HISTORY_TEMPLATE_NAME, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME
-    };
+
     public static final String[] TEMPLATE_NAMES_NO_ILM = new String[] {
         HISTORY_TEMPLATE_NAME_NO_ILM, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME
     };