Watcher: Fix SmokeTestWatcherWithSecurityIT test (elastic/x-pack-elasticsearch#4140)

These tests were configured to run every second, some of the watch
history entries could have an execution state throttled, when two
executions happened while waiting. The tests who were affected by 
this are now explicitely searching for the watch history entry with 
the correct state.

relates elastic/x-pack-elasticsearch#4137

Original commit: elastic/x-pack-elasticsearch@e463cc4770

qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

@@ -7,7 +7,6 @@ package org.elasticsearch.smoketest;
 
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
@@ -31,7 +30,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/4137")
 public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
 
     private static final String TEST_ADMIN_USERNAME = "test_admin";
@@ -121,9 +119,7 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
         }
 
         // check history, after watch has fired
-        ObjectPath objectPath = getWatchHistoryEntry("my_watch");
+        ObjectPath objectPath = getWatchHistoryEntry("my_watch", "executed");
-        String state = objectPath.evaluate("hits.hits.0._source.state");
-        assertThat(state, is("executed"));
         boolean conditionMet = objectPath.evaluate("hits.hits.0._source.result.condition.met");
         assertThat(conditionMet, is(true));
     }
@@ -174,9 +170,7 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
         }
 
         // check history, after watch has fired
-        ObjectPath objectPath = getWatchHistoryEntry("my_watch");
+        ObjectPath objectPath = getWatchHistoryEntry("my_watch", "executed");
-        String state = objectPath.evaluate("hits.hits.0._source.state");
-        assertThat(state, is("executed"));
         boolean conditionMet = objectPath.evaluate("hits.hits.0._source.result.condition.met");
         assertThat(conditionMet, is(true));
 
@@ -228,10 +222,7 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
             indexWatch("my_watch", builder);
         }
 
-        ObjectPath objectPath = getWatchHistoryEntry("my_watch");
+        ObjectPath objectPath = getWatchHistoryEntry("my_watch", "executed");
-
-        String state = objectPath.evaluate("hits.hits.0._source.state");
-        assertThat(state, is("executed"));
         boolean conditionMet = objectPath.evaluate("hits.hits.0._source.result.condition.met");
         assertThat(conditionMet, is(true));
 
@@ -255,10 +246,7 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
             indexWatch("my_watch", builder);
         }
 
-        ObjectPath objectPath = getWatchHistoryEntry("my_watch");
+        ObjectPath objectPath = getWatchHistoryEntry("my_watch", "executed");
-
-        String state = objectPath.evaluate("hits.hits.0._source.state");
-        assertThat(state, is("executed"));
         boolean conditionMet = objectPath.evaluate("hits.hits.0._source.result.condition.met");
         assertThat(conditionMet, is(true));
 
@@ -277,6 +265,10 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
     }
 
     private ObjectPath getWatchHistoryEntry(String watchId) throws Exception {
+        return getWatchHistoryEntry(watchId, null);
+    }
+
+    private ObjectPath getWatchHistoryEntry(String watchId, String state) throws Exception {
         final AtomicReference<ObjectPath> objectPathReference = new AtomicReference<>();
         assertBusy(() -> {
             client().performRequest("POST", ".watcher-history-*/_refresh");
@@ -286,6 +278,10 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
                 builder.startObject("query").startObject("bool").startArray("must");
                 builder.startObject().startObject("term").startObject("watch_id").field("value", watchId).endObject().endObject()
                         .endObject();
+                if (Strings.isNullOrEmpty(state) == false) {
+                    builder.startObject().startObject("term").startObject("state").field("value", state).endObject().endObject()
+                            .endObject();
+                }
                 builder.endArray().endObject().endObject();
                 builder.startArray("sort").startObject().startObject("trigger_event.triggered_time").field("order", "desc").endObject()
                         .endObject().endArray();