add a check that ensures we see the number of shards that we would expect in the cluster state before the test starts and retry after a pause if we do not

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1236366 13f79535-47bb-0310-9956-ffa450edef68

solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java

@@ -66,6 +66,8 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
   
   protected static final String DEFAULT_COLLECTION = "collection1";
   
+  private boolean printLayoutOnTearDown = false;
+  
   String t1 = "a_t";
   String i1 = "a_si";
   String nint = "n_i";
@@ -225,13 +227,25 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
     System.clearProperty("collection");
     controlClient = createNewSolrServer(controlJetty.getLocalPort());
     
-    createJettys(numServers);
+    createJettys(numServers, true);
     
   }
   
-  private List<JettySolrRunner> createJettys(int numJettys) throws Exception,
-      InterruptedException, TimeoutException, IOException, KeeperException,
-      URISyntaxException {
+  private List<JettySolrRunner> createJettys(int numJettys) throws Exception {
+    return createJettys(numJettys, false);
+  }
+  
+
+  /**
+   * @param numJettys
+   * @param checkCreatedVsState
+   *          if true, make sure the number created (numJettys) matches the
+   *          number in the cluster state - if you add more jetties this may not
+   *          be the case
+   * @return
+   * @throws Exception
+   */
+  private List<JettySolrRunner> createJettys(int numJettys, boolean checkCreatedVsState) throws Exception {
     List<JettySolrRunner> jettys = new ArrayList<JettySolrRunner>();
     List<SolrServer> clients = new ArrayList<SolrServer>();
     StringBuilder sb = new StringBuilder();
@@ -249,6 +263,23 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
     this.jettys.addAll(jettys);
     this.clients.addAll(clients);
     
+    if (checkCreatedVsState) {
+      // now wait until we see that the number of shards in the cluster state
+      // matches what we expect
+      int numShards = getNumShards(DEFAULT_COLLECTION);
+      int retries = 0;
+      while (numShards != shardCount) {
+        numShards = getNumShards(DEFAULT_COLLECTION);
+        if (numShards == shardCount) break;
+        if (retries++ == 20) {
+          printLayoutOnTearDown = true;
+          fail("Shards in the state does not match what we set:" + numShards
+              + " vs " + shardCount);
+        }
+        Thread.sleep(500);
+      }
+    }
+
     updateMappingsFromZk(this.jettys, this.clients);
     
     // build the shard string
@@ -263,6 +294,16 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
     
     return jettys;
   }
+
+  private int getNumShards(String defaultCollection) {
+    Map<String,Slice> slices = this.zkStateReader.getCloudState().getSlices(defaultCollection);
+    int cnt = 0;
+    for (Map.Entry<String,Slice> entry : slices.entrySet()) {
+      cnt += entry.getValue().getShards().size();
+    }
+    
+    return cnt;
+  }
   
   public JettySolrRunner createJetty(String dataDir, String shardList,
       String solrConfigOverride) throws Exception {
@@ -540,7 +581,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
       testFinished = true;
     } finally {
       if (!testFinished) {
-        printLayout();
+        printLayoutOnTearDown = true;
       }
     }
     
@@ -1194,7 +1235,7 @@ public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
   @Override
   @After
   public void tearDown() throws Exception {
-    if (VERBOSE) {
+    if (VERBOSE || printLayoutOnTearDown) {
       super.printLayout();
     }
     ((CommonsHttpSolrServer) controlClient).shutdown();