diff --git a/src/main/java/org/elasticsearch/alerts/AlertsLifeCycleService.java b/src/main/java/org/elasticsearch/alerts/AlertsLifeCycleService.java
index a1721e53fec..5657e47545a 100644
--- a/src/main/java/org/elasticsearch/alerts/AlertsLifeCycleService.java
+++ b/src/main/java/org/elasticsearch/alerts/AlertsLifeCycleService.java
@@ -7,6 +7,7 @@ package org.elasticsearch.alerts;
 
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.LifecycleListener;
@@ -39,19 +40,26 @@ public class AlertsLifeCycleService extends AbstractComponent implements Cluster
         indicesService.addLifecycleListener(new LifecycleListener() {
             @Override
             public void beforeStop() {
-                AlertsLifeCycleService.this.alertsService.stop();
+                stop(false);
             }
         });
         manuallyStopped = !settings.getAsBoolean("alerts.start_immediately",  true);
     }
 
-    public synchronized void start() {
-        manuallyStopped = false;
-        alertsService.start(clusterService.state());
+    public void start() {
+        start(clusterService.state());
     }
 
-    public synchronized void stop() {
-        manuallyStopped = true;
+    public void stop() {
+        stop(true);
+    }
+
+    private synchronized void start(ClusterState state) {
+        alertsService.start(state);
+    }
+
+    private synchronized void stop(boolean manual) {
+        manuallyStopped = manual;
         alertsService.stop();
     }
 
@@ -65,7 +73,7 @@ public class AlertsLifeCycleService extends AbstractComponent implements Cluster
             threadPool.executor(ThreadPool.Names.GENERIC).execute(new Runnable() {
                 @Override
                 public void run() {
-                    alertsService.stop();
+                    stop(false);
                 }
             });
         } else {
@@ -78,7 +86,7 @@ public class AlertsLifeCycleService extends AbstractComponent implements Cluster
                 threadPool.executor(ThreadPool.Names.GENERIC).execute(new Runnable() {
                     @Override
                     public void run() {
-                        alertsService.start(event.state());
+                        start(event.state());
                     }
                 });
             }
diff --git a/src/test/java/org/elasticsearch/alerts/test/AbstractAlertsSingleNodeTests.java b/src/test/java/org/elasticsearch/alerts/test/AbstractAlertsSingleNodeTests.java
index 23858d4e304..2128a5818f8 100644
--- a/src/test/java/org/elasticsearch/alerts/test/AbstractAlertsSingleNodeTests.java
+++ b/src/test/java/org/elasticsearch/alerts/test/AbstractAlertsSingleNodeTests.java
@@ -9,7 +9,6 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.alerts.AlertsService;
 import org.elasticsearch.alerts.support.init.proxy.ClientProxy;
 import org.elasticsearch.alerts.support.init.proxy.ScriptServiceProxy;
 import org.elasticsearch.client.Requests;
@@ -31,19 +30,7 @@ public abstract class AbstractAlertsSingleNodeTests extends ElasticsearchSingleN
 
     @AfterClass
     public static void cleanupSuite() throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(getInstanceFromNode(AlertsService.class).state(), equalTo(AlertsService.State.STARTED));
-            }
-        });
         node().stop();
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(getInstanceFromNode(AlertsService.class).state(), equalTo(AlertsService.State.STOPPED));
-            }
-        });
     }
 
     @Override