Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18185 IntegrationTestTimeBoundedRequestsWithRegionReplicas unbalanced tests fails with AssertionError 

unbalance.kill.meta.rs property was added which controls the monkey to
kill that region server which holds hbase:meta.

Change-Id: I049d05cc13a78ba868c917879f67ef4f34fffbf9
Signed-off-by: Apekshit Sharma <appy@apache.org>
This commit is contained in:
Balazs Meszaros 2017-06-28 11:17:12 +02:00 committed by Apekshit Sharma
parent 9a1661832d
commit c5d3de0cd4
3 changed files with 26 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceKillAndRebalanceAction.java
View File

@ -41,12 +41,15 @@ public class UnbalanceKillAndRebalanceAction extends Action {
private long waitForUnbalanceMilliSec;
private long waitForKillsMilliSec;
private long waitAfterBalanceMilliSec;
private boolean killMetaRs;
public UnbalanceKillAndRebalanceAction(long waitUnbalance, long waitKill, long waitAfterBalance) {
public UnbalanceKillAndRebalanceAction(long waitUnbalance, long waitKill, long waitAfterBalance,
boolean killMetaRs) {
super();
waitForUnbalanceMilliSec = waitUnbalance;
waitForKillsMilliSec = waitKill;
waitAfterBalanceMilliSec = waitAfterBalance;
this.killMetaRs = killMetaRs;
}
@Override
@ -57,7 +60,9 @@ public class UnbalanceKillAndRebalanceAction extends Action {
int liveCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_LIVE * victimServers.size());
int deadCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_DIE * victimServers.size());
Assert.assertTrue((liveCount + deadCount) < victimServers.size());
Assert.assertTrue(
"There are not enough victim servers: " + victimServers.size(),
liveCount + deadCount < victimServers.size());
List<ServerName> targetServers = new ArrayList<>(liveCount);
for (int i = 0; i < liveCount + deadCount; ++i) {
int victimIx = RandomUtils.nextInt(victimServers.size());
@ -65,14 +70,23 @@ public class UnbalanceKillAndRebalanceAction extends Action {
}
unbalanceRegions(status, victimServers, targetServers, HOARD_FRC_OF_REGIONS);
Thread.sleep(waitForUnbalanceMilliSec);
for (int i = 0; i < liveCount; ++i) {
ServerName metaServer = cluster.getServerHoldingMeta();
for (ServerName targetServer: targetServers) {
// Don't keep killing servers if we're
// trying to stop the monkey.
if (context.isStopping()) {
break;
}
killRs(targetServers.get(i));
killedServers.add(targetServers.get(i));
if (killedServers.size() >= liveCount) {
break;
}
if (!killMetaRs && targetServer.equals(metaServer)) {
LOG.info("Not killing server because it holds hbase:meta.");
} else {
killRs(targetServer);
killedServers.add(targetServer);
}
}
Thread.sleep(waitForKillsMilliSec);

2
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
View File

@ -39,6 +39,7 @@ public interface MonkeyConstants {
String UNBALANCE_WAIT_FOR_UNBALANCE_MS = "unbalance.action.wait.period";
String UNBALANCE_WAIT_FOR_KILLS_MS = "unbalance.action.kill.period";
String UNBALANCE_WAIT_AFTER_BALANCE_MS = "unbalance.action.wait.after.period";
String UNBALANCE_KILL_META_RS = "unbalance.action.kill.meta.rs";
String DECREASE_HFILE_SIZE_SLEEP_TIME = "decrease.hfile.size.sleep.time";
long DEFAULT_PERIODIC_ACTION1_PERIOD = 60 * 1000;
@ -61,5 +62,6 @@ public interface MonkeyConstants {
long DEFAULT_UNBALANCE_WAIT_FOR_UNBALANCE_MS = 2 * 1000;
long DEFAULT_UNBALANCE_WAIT_FOR_KILLS_MS = 2 * 1000;
long DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS = 5 * 1000;
boolean DEFAULT_UNBALANCE_KILL_META_RS = true;
long DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME = 30 * 1000;
}

6
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/UnbalanceMonkeyFactory.java
View File

@ -31,13 +31,14 @@ public class UnbalanceMonkeyFactory extends MonkeyFactory {
private long waitForUnbalanceMilliSec;
private long waitForKillMilliSec;
private long waitAfterBalanceMilliSec;
private boolean killMetaRs;
@Override
public ChaosMonkey build() {
loadProperties();
Policy chaosPolicy = new PeriodicRandomActionPolicy(chaosEveryMilliSec,
new UnbalanceKillAndRebalanceAction(waitForUnbalanceMilliSec, waitForKillMilliSec,
waitAfterBalanceMilliSec));
waitAfterBalanceMilliSec, killMetaRs));
return new PolicyBasedChaosMonkey(util, chaosPolicy);
}
@ -55,5 +56,8 @@ public class UnbalanceMonkeyFactory extends MonkeyFactory {
waitAfterBalanceMilliSec = Long.parseLong(this.properties.getProperty(
MonkeyConstants.UNBALANCE_WAIT_AFTER_BALANCE_MS,
MonkeyConstants.DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS + ""));
killMetaRs = Boolean.parseBoolean(this.properties.getProperty(
MonkeyConstants.UNBALANCE_KILL_META_RS,
MonkeyConstants.DEFAULT_UNBALANCE_KILL_META_RS + ""));
}
}