diff --git a/src/test/java/org/elasticsearch/alerts/BootStrapTest.java b/src/test/java/org/elasticsearch/alerts/BootStrapTest.java
index b2408464760..ed3deb26585 100644
--- a/src/test/java/org/elasticsearch/alerts/BootStrapTest.java
+++ b/src/test/java/org/elasticsearch/alerts/BootStrapTest.java
@@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsEqual.equalTo;
 
@@ -107,9 +108,6 @@ public class BootStrapTest extends AbstractAlertingTests {
                 .get();
         assertTrue(indexResponse.isCreated());
 
-        // The executor doesn't get shutdown when alerts stops, so the largest queue size don't get reset.
-        long previousLargestQueueSize = response.getAlertActionManagerLargestQueueSize();
-
         stopAlerting();
         startAlerting();
 
@@ -117,7 +115,7 @@ public class BootStrapTest extends AbstractAlertingTests {
         assertTrue(response.isAlertActionManagerStarted());
         assertThat(response.getAlertManagerStarted(), equalTo(AlertsService.State.STARTED));
         assertThat(response.getNumberOfRegisteredAlerts(), equalTo(1L));
-        assertThat(response.getAlertActionManagerLargestQueueSize() - previousLargestQueueSize, equalTo(1L));
+        assertThat(response.getAlertActionManagerLargestQueueSize(), greaterThanOrEqualTo(1l));
     }
 
     @Test