tests: have ChaosMonkeyNothingIsSafeTest start up down nodes at the end so that any final updates won't fail because there is no leader coming

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1545409 13f79535-47bb-0310-9956-ffa450edef68

solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java

@@ -138,7 +138,7 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
         ftIndexThread.start();
       }
       
-      chaosMonkey.startTheMonkey(true, 10000);
+      chaosMonkey.startTheMonkey(true, true, 10000);
 
       long runLength;
       if (RUN_LENGTH != -1) {
@@ -158,18 +158,13 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
         indexThread.safeStop();
       }
       
+      // start any downed jetties to be sure we still will end up with a leader per shard...
+      
       // wait for stop...
       for (StopableThread indexThread : threads) {
         indexThread.join();
       }
       
-       // we expect full throttle fails, but cloud client should not easily fail
-       for (StopableThread indexThread : threads) {
-         if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
-           assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
-         }
-       }
-      
       // try and wait for any replications and what not to finish...
       
       Thread.sleep(2000);
@@ -190,6 +185,13 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
       assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);
       
       
+      // we expect full throttle fails, but cloud client should not easily fail
+      for (StopableThread indexThread : threads) {
+        if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
+          assertEquals("There were expected update fails", 0, ((StopableIndexingThread) indexThread).getFails());
+        }
+      }
+      
       // full throttle thread can
       // have request fails 
       checkShardConsistency(!runFullThrottle, true);

solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java

@@ -99,7 +99,7 @@ public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
       indexThread.start();
     }
     
-    chaosMonkey.startTheMonkey(false, 500);
+    chaosMonkey.startTheMonkey(false, false, 500);
     long runLength;
     if (RUN_LENGTH != -1) {
       runLength = RUN_LENGTH;

solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java

@@ -435,7 +435,7 @@ public class ChaosMonkey {
   
   // synchronously starts and stops shards randomly, unless there is only one
   // active shard up for a slice or if there is one active and others recovering
-  public void startTheMonkey(boolean killLeaders, final int roundPauseUpperLimit) {
+  public void startTheMonkey(boolean killLeaders, final boolean startDeadPool, final int roundPauseUpperLimit) {
     if (!MONKEY_ENABLED) {
       monkeyLog("The Monkey is disabled and will not start");
       return;
@@ -503,6 +503,16 @@ public class ChaosMonkey {
         monkeyLog("I ran for " + (System.currentTimeMillis() - startTime)/1000.0f + "sec. I stopped " + stops + " and I started " + starts
             + ". I also expired " + expires.get() + " and caused " + connloss
             + " connection losses");
+        if (startDeadPool) {
+          // starting down nodes
+          for (CloudJettyRunner jetty : deadPool) {
+            try {
+              if (jetty.jetty.isStopped()) ChaosMonkey.start(jetty.jetty);
+            } catch (Exception e) {
+              log.error("", e);
+            }
+          }
+        }
       }
     };
     monkeyThread.start();