wait for *all* nodes to see downed node out of cloud state before making a request

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1365428 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2012-07-25 03:47:09 +00:00
parent c5f10f722b
commit 3208495042
1 changed files with 18 additions and 15 deletions

View File

@ -150,25 +150,16 @@ public class SyncSliceTest extends FullSolrCloudTest {
chaosMonkey.killJetty(leaderJetty);
CloudJettyRunner upJetty = jetties.iterator().next();
// we are careful to make sure the downed node is no longer in the state,
// because on some systems (especially freebsd w/ blackhole enabled), trying
// to talk to a downed node causes grief
assertNotNull(upJetty.jetty.getDispatchFilter());
assertNotNull(upJetty.jetty.getDispatchFilter());
assertNotNull(upJetty.jetty.getDispatchFilter().getFilter());
int tries = 0;
while (((SolrDispatchFilter) upJetty.jetty.getDispatchFilter().getFilter())
.getCores().getZkController().getZkStateReader().getCloudState()
.liveNodesContain(leaderJetty.info.get(ZkStateReader.NODE_NAME_PROP))) {
if (tries++ == 120) {
fail("Shard still reported as live in zk");
}
Thread.sleep(1000);
for (CloudJettyRunner cjetty : jetties) {
waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
.getFilter()).getCores().getZkController().getZkStateReader(),
leaderJetty);
}
waitToSeeNotLive(cloudClient.getZkStateReader(), leaderJetty);
waitForThingsToLevelOut();
checkShardConsistency(false, true);
@ -177,6 +168,18 @@ public class SyncSliceTest extends FullSolrCloudTest {
assertEquals(5, cloudClientDocs);
}
private void waitToSeeNotLive(ZkStateReader zkStateReader,
CloudJettyRunner cjetty) throws InterruptedException {
int tries = 0;
while (zkStateReader.getCloudState()
.liveNodesContain(cjetty.info.get(ZkStateReader.NODE_NAME_PROP))) {
if (tries++ == 120) {
fail("Shard still reported as live in zk");
}
Thread.sleep(1000);
}
}
private void waitForThingsToLevelOut() throws Exception {
int cnt = 0;
boolean retry = false;