diff --git a/src/main/java/org/elasticsearch/alerts/AlertsService.java b/src/main/java/org/elasticsearch/alerts/AlertsService.java
index 15e43dd9e43..11abe5a3eec 100644
--- a/src/main/java/org/elasticsearch/alerts/AlertsService.java
+++ b/src/main/java/org/elasticsearch/alerts/AlertsService.java
@@ -64,6 +64,14 @@ public class AlertsService extends AbstractComponent {
         manuallyStopped = !settings.getAsBoolean("alerts.start_immediately",  true);
     }
 
+    // TODO: consider making this adding start/stop lock
+    // The currently mechanism isn't broken, but in tests it is annoying that if stop has been invoked concurrently
+    // and the first invocation sets the state to STOPPING then the second invocation will just return, because the
+    // first invocation will do the work to stop alerts plugin. If the second invocation was caused by a test teardown
+    // then the thread lead detection will fail, because it assumes that everything should have been stopped & closed.
+    // This isn't the case in the situation, although this will happen, but to late for the thread leak detection to
+    // not see it as a failure.
+
     /**
      * Manually starts alerting if not already started
      */
diff --git a/src/test/java/org/elasticsearch/alerts/test/AbstractAlertsSingleNodeTests.java b/src/test/java/org/elasticsearch/alerts/test/AbstractAlertsSingleNodeTests.java
index b03ad50108e..af445c15dc2 100644
--- a/src/test/java/org/elasticsearch/alerts/test/AbstractAlertsSingleNodeTests.java
+++ b/src/test/java/org/elasticsearch/alerts/test/AbstractAlertsSingleNodeTests.java
@@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.alerts.AlertsService;
 import org.elasticsearch.alerts.support.init.proxy.ClientProxy;
 import org.elasticsearch.alerts.support.init.proxy.ScriptServiceProxy;
 import org.elasticsearch.client.Requests;
@@ -31,6 +32,12 @@ public abstract class AbstractAlertsSingleNodeTests extends ElasticsearchSingleN
     @AfterClass
     public static void cleanupSuite() throws Exception {
         node().stop();
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                assertThat(getInstanceFromNode(AlertsService.class).state(), equalTo(AlertsService.State.STOPPED));
+            }
+        });
     }
 
     @Override