Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18185 IntegrationTestTimeBoundedRequestsWithRegionReplicas unbalanced tests fails with AssertionError 

unbalance.kill.meta.rs property was added which controls the monkey to
kill that region server which holds hbase:meta.

Change-Id: I2c871789645b6c1986104f5a16cc6b9badfbc172
Signed-off-by: Apekshit Sharma <appy@apache.org>
This commit is contained in:
Balazs Meszaros 2017-06-28 11:17:12 +02:00 committed by Apekshit Sharma
parent e662d04321
commit 0053cb967b
3 changed files with 26 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceKillAndRebalanceAction.java
View File

@ -41,12 +41,15 @@ public class UnbalanceKillAndRebalanceAction extends Action {
private long waitForUnbalanceMilliSec; private long waitForUnbalanceMilliSec;
private long waitForKillsMilliSec; private long waitForKillsMilliSec;
private long waitAfterBalanceMilliSec; private long waitAfterBalanceMilliSec;
private boolean killMetaRs;
public UnbalanceKillAndRebalanceAction(long waitUnbalance, long waitKill, long waitAfterBalance) { public UnbalanceKillAndRebalanceAction(long waitUnbalance, long waitKill, long waitAfterBalance,
boolean killMetaRs) {
super(); super();
waitForUnbalanceMilliSec = waitUnbalance; waitForUnbalanceMilliSec = waitUnbalance;
waitForKillsMilliSec = waitKill; waitForKillsMilliSec = waitKill;
waitAfterBalanceMilliSec = waitAfterBalance; waitAfterBalanceMilliSec = waitAfterBalance;
this.killMetaRs = killMetaRs;
} }
@Override @Override
@ -57,7 +60,9 @@ public class UnbalanceKillAndRebalanceAction extends Action {
int liveCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_LIVE * victimServers.size()); int liveCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_LIVE * victimServers.size());
int deadCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_DIE * victimServers.size()); int deadCount = (int)Math.ceil(FRC_SERVERS_THAT_HOARD_AND_DIE * victimServers.size());
Assert.assertTrue((liveCount + deadCount) < victimServers.size()); Assert.assertTrue(
"There are not enough victim servers: " + victimServers.size(),
liveCount + deadCount < victimServers.size());
List<ServerName> targetServers = new ArrayList<ServerName>(liveCount); List<ServerName> targetServers = new ArrayList<ServerName>(liveCount);
for (int i = 0; i < liveCount + deadCount; ++i) { for (int i = 0; i < liveCount + deadCount; ++i) {
int victimIx = RandomUtils.nextInt(victimServers.size()); int victimIx = RandomUtils.nextInt(victimServers.size());
@ -65,14 +70,23 @@ public class UnbalanceKillAndRebalanceAction extends Action {
} }
unbalanceRegions(status, victimServers, targetServers, HOARD_FRC_OF_REGIONS); unbalanceRegions(status, victimServers, targetServers, HOARD_FRC_OF_REGIONS);
Thread.sleep(waitForUnbalanceMilliSec); Thread.sleep(waitForUnbalanceMilliSec);
for (int i = 0; i < liveCount; ++i) { ServerName metaServer = cluster.getServerHoldingMeta();
for (ServerName targetServer: targetServers) {
// Don't keep killing servers if we're // Don't keep killing servers if we're
// trying to stop the monkey. // trying to stop the monkey.
if (context.isStopping()) { if (context.isStopping()) {
break; break;
} }
killRs(targetServers.get(i)); if (killedServers.size() >= liveCount) {
killedServers.add(targetServers.get(i)); break;
}
if (!killMetaRs && targetServer.equals(metaServer)) {
LOG.info("Not killing server because it holds hbase:meta.");
} else {
killRs(targetServer);
killedServers.add(targetServer);
}
} }
Thread.sleep(waitForKillsMilliSec); Thread.sleep(waitForKillsMilliSec);

2
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
View File

@ -39,6 +39,7 @@ public interface MonkeyConstants {
String UNBALANCE_WAIT_FOR_UNBALANCE_MS = "unbalance.action.wait.period"; String UNBALANCE_WAIT_FOR_UNBALANCE_MS = "unbalance.action.wait.period";
String UNBALANCE_WAIT_FOR_KILLS_MS = "unbalance.action.kill.period"; String UNBALANCE_WAIT_FOR_KILLS_MS = "unbalance.action.kill.period";
String UNBALANCE_WAIT_AFTER_BALANCE_MS = "unbalance.action.wait.after.period"; String UNBALANCE_WAIT_AFTER_BALANCE_MS = "unbalance.action.wait.after.period";
String UNBALANCE_KILL_META_RS = "unbalance.action.kill.meta.rs";
String DECREASE_HFILE_SIZE_SLEEP_TIME = "decrease.hfile.size.sleep.time"; String DECREASE_HFILE_SIZE_SLEEP_TIME = "decrease.hfile.size.sleep.time";
long DEFAULT_PERIODIC_ACTION1_PERIOD = 60 * 1000; long DEFAULT_PERIODIC_ACTION1_PERIOD = 60 * 1000;
@ -61,5 +62,6 @@ public interface MonkeyConstants {
long DEFAULT_UNBALANCE_WAIT_FOR_UNBALANCE_MS = 2 * 1000; long DEFAULT_UNBALANCE_WAIT_FOR_UNBALANCE_MS = 2 * 1000;
long DEFAULT_UNBALANCE_WAIT_FOR_KILLS_MS = 2 * 1000; long DEFAULT_UNBALANCE_WAIT_FOR_KILLS_MS = 2 * 1000;
long DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS = 5 * 1000; long DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS = 5 * 1000;
boolean DEFAULT_UNBALANCE_KILL_META_RS = true;
long DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME = 30 * 1000; long DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME = 30 * 1000;
} }

6
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/UnbalanceMonkeyFactory.java
View File

@ -31,13 +31,14 @@ public class UnbalanceMonkeyFactory extends MonkeyFactory {
private long waitForUnbalanceMilliSec; private long waitForUnbalanceMilliSec;
private long waitForKillMilliSec; private long waitForKillMilliSec;
private long waitAfterBalanceMilliSec; private long waitAfterBalanceMilliSec;
private boolean killMetaRs;
@Override @Override
public ChaosMonkey build() { public ChaosMonkey build() {
loadProperties(); loadProperties();
Policy chaosPolicy = new PeriodicRandomActionPolicy(chaosEveryMilliSec, Policy chaosPolicy = new PeriodicRandomActionPolicy(chaosEveryMilliSec,
new UnbalanceKillAndRebalanceAction(waitForUnbalanceMilliSec, waitForKillMilliSec, new UnbalanceKillAndRebalanceAction(waitForUnbalanceMilliSec, waitForKillMilliSec,
waitAfterBalanceMilliSec)); waitAfterBalanceMilliSec, killMetaRs));
return new PolicyBasedChaosMonkey(util, chaosPolicy); return new PolicyBasedChaosMonkey(util, chaosPolicy);
} }
@ -55,5 +56,8 @@ public class UnbalanceMonkeyFactory extends MonkeyFactory {
waitAfterBalanceMilliSec = Long.parseLong(this.properties.getProperty( waitAfterBalanceMilliSec = Long.parseLong(this.properties.getProperty(
MonkeyConstants.UNBALANCE_WAIT_AFTER_BALANCE_MS, MonkeyConstants.UNBALANCE_WAIT_AFTER_BALANCE_MS,
MonkeyConstants.DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS + "")); MonkeyConstants.DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS + ""));
killMetaRs = Boolean.parseBoolean(this.properties.getProperty(
MonkeyConstants.UNBALANCE_KILL_META_RS,
MonkeyConstants.DEFAULT_UNBALANCE_KILL_META_RS + ""));
} }
} }