diff --git a/src/main/java/org/elasticsearch/alerts/AlertsStore.java b/src/main/java/org/elasticsearch/alerts/AlertsStore.java
index 21524168f89..29c166c9126 100644
--- a/src/main/java/org/elasticsearch/alerts/AlertsStore.java
+++ b/src/main/java/org/elasticsearch/alerts/AlertsStore.java
@@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexRequest;
@@ -196,7 +197,11 @@ public class AlertsStore extends AbstractComponent {
 
     private void loadAlerts() {
         assert alertMap.isEmpty() : "No alerts should reside, but there are " + alertMap.size() + " alerts.";
-        client.admin().indices().refresh(new RefreshRequest(ALERT_INDEX)).actionGet();
+        RefreshResponse refreshResponse = client.admin().indices().refresh(new RefreshRequest(ALERT_INDEX)).actionGet();
+        if (refreshResponse.getSuccessfulShards() != refreshResponse.getSuccessfulShards()) {
+            throw new ElasticsearchException("Not all shards have been refreshed");
+        }
+
         SearchResponse response = client.prepareSearch(ALERT_INDEX)
                 .setTypes(ALERT_TYPE)
                 .setSearchType(SearchType.SCAN)
diff --git a/src/main/java/org/elasticsearch/alerts/actions/AlertActionManager.java b/src/main/java/org/elasticsearch/alerts/actions/AlertActionManager.java
index 3bf09c30e6b..4293c5be70d 100644
--- a/src/main/java/org/elasticsearch/alerts/actions/AlertActionManager.java
+++ b/src/main/java/org/elasticsearch/alerts/actions/AlertActionManager.java
@@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
@@ -162,7 +163,11 @@ public class AlertActionManager extends AbstractComponent {
 
     public void loadQueue() {
         assert actionsToBeProcessed.isEmpty() : "Queue should be empty, but contains " + actionsToBeProcessed.size() + " elements.";
-        client.admin().indices().refresh(new RefreshRequest(ALERT_HISTORY_INDEX_PREFIX + "*")).actionGet();
+        RefreshResponse refreshResponse = client.admin().indices().refresh(new RefreshRequest(ALERT_HISTORY_INDEX_PREFIX + "*")).actionGet();
+        if (refreshResponse.getSuccessfulShards() != refreshResponse.getSuccessfulShards()) {
+            throw new ElasticsearchException("Not all shards have been refreshed");
+        }
+
         SearchResponse response = client.prepareSearch(ALERT_HISTORY_INDEX_PREFIX + "*")
                 .setQuery(QueryBuilders.termQuery(STATE, AlertActionState.SEARCH_NEEDED.toString()))
                 .setSearchType(SearchType.SCAN)