diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java index 2a5e1d0a4a5..639977df890 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java @@ -21,6 +21,7 @@ import java.io.File; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; @@ -136,6 +137,8 @@ public class TestMiniSolrCloudCluster extends LuceneTestCase { SolrInputDocument doc = new SolrInputDocument(); doc.setField("id", "1"); + ZkStateReader zkStateReader = new ZkStateReader(zkClient); + waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330); cloudSolrServer.add(doc); cloudSolrServer.commit(); SolrQuery query = new SolrQuery(); @@ -144,7 +147,6 @@ public class TestMiniSolrCloudCluster extends LuceneTestCase { assertEquals(1, rsp.getResults().getNumFound()); // remove a server not hosting any replicas - ZkStateReader zkStateReader = new ZkStateReader(zkClient); zkStateReader.updateClusterState(true); ClusterState clusterState = zkStateReader.getClusterState(); HashMap jettyMap = new HashMap(); @@ -226,4 +228,55 @@ public class TestMiniSolrCloudCluster extends LuceneTestCase { request.setPath("/admin/collections"); return server.request(request); } + + protected void waitForRecoveriesToFinish(String collection, + ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout, int timeoutSeconds) + throws Exception { + log.info("Wait for recoveries to finish - collection: " + collection + " failOnTimeout:" + failOnTimeout + " timeout (sec):" + timeoutSeconds); + boolean cont = true; + int cnt = 0; + + while (cont) { + if (verbose) System.out.println("-"); + boolean sawLiveRecovering = false; + zkStateReader.updateClusterState(true); + ClusterState clusterState = zkStateReader.getClusterState(); + Map slices = clusterState.getSlicesMap(collection); + assertNotNull("Could not find collection:" + collection, slices); + for (Map.Entry entry : slices.entrySet()) { + Map shards = entry.getValue().getReplicasMap(); + for (Map.Entry shard : shards.entrySet()) { + if (verbose) System.out.println("rstate:" + + shard.getValue().getStr(ZkStateReader.STATE_PROP) + + " live:" + + clusterState.liveNodesContain(shard.getValue().getNodeName())); + String state = shard.getValue().getStr(ZkStateReader.STATE_PROP); + if ((state.equals(ZkStateReader.RECOVERING) || state + .equals(ZkStateReader.SYNC) || state.equals(ZkStateReader.DOWN)) + && clusterState.liveNodesContain(shard.getValue().getStr( + ZkStateReader.NODE_NAME_PROP))) { + sawLiveRecovering = true; + } + } + } + if (!sawLiveRecovering || cnt == timeoutSeconds) { + if (!sawLiveRecovering) { + if (verbose) System.out.println("no one is recoverying"); + } else { + if (verbose) System.out.println("Gave up waiting for recovery to finish.."); + if (failOnTimeout) { + fail("There are still nodes recoverying - waited for " + timeoutSeconds + " seconds"); + // won't get here + return; + } + } + cont = false; + } else { + Thread.sleep(1000); + } + cnt++; + } + + log.info("Recoveries finished - collection: " + collection); + } }