Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tests: have ChaosMonkeyNothingIsSafeTest start up down nodes at the end so that any final updates won't fail because there is no leader coming 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1545409 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-11-25 21:14:11 +00:00
parent dc147b8ee0
commit 295ce29bfb
3 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
View File

@ -138,7 +138,7 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
ftIndexThread.start();
}
chaosMonkey.startTheMonkey(true, 10000);
chaosMonkey.startTheMonkey(true, true, 10000);
long runLength;
if (RUN_LENGTH != -1) {
@ -158,18 +158,13 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
indexThread.safeStop();
}
// start any downed jetties to be sure we still will end up with a leader per shard...
// wait for stop...
for (StopableThread indexThread : threads) {
indexThread.join();
}
// we expect full throttle fails, but cloud client should not easily fail
for (StopableThread indexThread : threads) {
if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
}
}
// try and wait for any replications and what not to finish...
Thread.sleep(2000);
@ -190,6 +185,13 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);
// we expect full throttle fails, but cloud client should not easily fail
for (StopableThread indexThread : threads) {
if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
assertEquals("There were expected update fails", 0, ((StopableIndexingThread) indexThread).getFails());
}
}
// full throttle thread can
// have request fails
checkShardConsistency(!runFullThrottle, true);

2
solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
View File

@ -99,7 +99,7 @@ public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
indexThread.start();
}
chaosMonkey.startTheMonkey(false, 500);
chaosMonkey.startTheMonkey(false, false, 500);
long runLength;
if (RUN_LENGTH != -1) {
runLength = RUN_LENGTH;

12
solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java
View File

@ -435,7 +435,7 @@ public class ChaosMonkey {
// synchronously starts and stops shards randomly, unless there is only one
// active shard up for a slice or if there is one active and others recovering
public void startTheMonkey(boolean killLeaders, final int roundPauseUpperLimit) {
public void startTheMonkey(boolean killLeaders, final boolean startDeadPool, final int roundPauseUpperLimit) {
if (!MONKEY_ENABLED) {
monkeyLog("The Monkey is disabled and will not start");
return;
@ -503,6 +503,16 @@ public class ChaosMonkey {
monkeyLog("I ran for " + (System.currentTimeMillis() - startTime)/1000.0f + "sec. I stopped " + stops + " and I started " + starts
+ ". I also expired " + expires.get() + " and caused " + connloss
+ " connection losses");
if (startDeadPool) {
// starting down nodes
for (CloudJettyRunner jetty : deadPool) {
try {
if (jetty.jetty.isStopped()) ChaosMonkey.start(jetty.jetty);
} catch (Exception e) {
log.error("", e);
}
}
}
}
};
monkeyThread.start();