HBASE-13157 Add Mob compaction actions and monkeys to Chaos Monkey

This commit is contained in:
Jonathan M Hsieh 2015-03-03 17:15:26 -08:00 committed by Jonathan M Hsieh
parent 45711ebaaf
commit 0d6cac9b1e
4 changed files with 295 additions and 0 deletions

View File

@ -0,0 +1,65 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.chaos.actions;
import org.apache.commons.lang.math.RandomUtils;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
/**
* Action that queues a table compaction.
*/
public class CompactMobAction extends Action {
private final TableName tableName;
private final int majorRatio;
private final long sleepTime;
public CompactMobAction(TableName tableName, float majorRatio) {
this(-1, tableName, majorRatio);
}
public CompactMobAction(
int sleepTime, TableName tableName, float majorRatio) {
this.tableName = tableName;
this.majorRatio = (int) (100 * majorRatio);
this.sleepTime = sleepTime;
}
@Override
public void perform() throws Exception {
HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
Admin admin = util.getHBaseAdmin();
boolean major = RandomUtils.nextInt(100) < majorRatio;
LOG.info("Performing action: Compact mob of table " + tableName + ", major=" + major);
try {
if (major) {
admin.majorCompactMob(tableName);
} else {
admin.compactMob(tableName);
}
} catch (Exception ex) {
LOG.warn("Mob Compaction failed, might be caused by other chaos: " + ex.getMessage());
}
if (sleepTime > 0) {
Thread.sleep(sleepTime);
}
}
}

View File

@ -0,0 +1,70 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.chaos.factories;
import org.apache.hadoop.hbase.chaos.actions.*;
import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
import org.apache.hadoop.hbase.chaos.policies.TwoConcurrentActionPolicy;
/**
* This is a copy of NoKillMonkeyFactory that also does mob compactions.
*/
public class MobNoKillMonkeyFactory extends MonkeyFactory {
@Override public ChaosMonkey build() {
Action[] actions1 = new Action[] {
new CompactMobAction(tableName, MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD),
new CompactTableAction(tableName, MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD),
new CompactRandomRegionOfTableAction(tableName,
MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO),
new FlushTableAction(tableName),
new FlushRandomRegionOfTableAction(tableName),
new MoveRandomRegionOfTableAction(tableName)
};
Action[] actions2 = new Action[] {
new SplitRandomRegionOfTableAction(tableName),
new MergeRandomAdjacentRegionsOfTableAction(tableName),
new SnapshotTableAction(tableName),
new AddColumnAction(tableName),
new RemoveColumnAction(tableName, columnFamilies),
new ChangeEncodingAction(tableName),
new ChangeCompressionAction(tableName),
new ChangeBloomFilterAction(tableName),
new ChangeVersionsAction(tableName)
};
Action[] actions3 = new Action[] {
new MoveRegionsOfTableAction(MonkeyConstants.DEFAULT_MOVE_REGIONS_SLEEP_TIME,
MonkeyConstants.DEFAULT_MOVE_REGIONS_MAX_TIME,
tableName),
new MoveRandomRegionOfTableAction(MonkeyConstants.DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME,
tableName),
};
Action[] actions4 = new Action[] {
new DumpClusterStatusAction()
};
return new PolicyBasedChaosMonkey(util,
new TwoConcurrentActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD, actions1, actions2),
new PeriodicRandomActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION2_PERIOD,actions3),
new PeriodicRandomActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION4_PERIOD,actions4));
}
}

View File

@ -0,0 +1,156 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.chaos.factories;
import org.apache.hadoop.hbase.chaos.actions.*;
import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy;
import org.apache.hadoop.hbase.chaos.policies.DoActionsOncePolicy;
import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
/**
* This is a copy of SlowDeterministicMonkeyFactory that also does mob compactions.
*/
public class MobSlowDeterministicMonkeyFactory extends MonkeyFactory {
private long action1Period;
private long action2Period;
private long action3Period;
private long action4Period;
private long moveRegionsMaxTime;
private long moveRegionsSleepTime;
private long moveRandomRegionSleepTime;
private long restartRandomRSSleepTime;
private long batchRestartRSSleepTime;
private float batchRestartRSRatio;
private long restartActiveMasterSleepTime;
private long rollingBatchRestartRSSleepTime;
private float rollingBatchRestartRSRatio;
private long restartRsHoldingMetaSleepTime;
private float compactTableRatio;
private float compactRandomRegionRatio;
@Override
public ChaosMonkey build() {
loadProperties();
// Actions such as compact/flush a table/region,
// move one region around. They are not so destructive,
// can be executed more frequently.
Action[] actions1 = new Action[] {
new CompactMobAction(tableName, compactTableRatio),
new CompactTableAction(tableName, compactTableRatio),
new CompactRandomRegionOfTableAction(tableName, compactRandomRegionRatio),
new FlushTableAction(tableName),
new FlushRandomRegionOfTableAction(tableName),
new MoveRandomRegionOfTableAction(tableName)
};
// Actions such as split/merge/snapshot.
// They should not cause data loss, or unreliability
// such as region stuck in transition.
Action[] actions2 = new Action[] {
new SplitRandomRegionOfTableAction(tableName),
new MergeRandomAdjacentRegionsOfTableAction(tableName),
new SnapshotTableAction(tableName),
new AddColumnAction(tableName),
new RemoveColumnAction(tableName, columnFamilies),
new ChangeEncodingAction(tableName),
new ChangeCompressionAction(tableName),
new ChangeBloomFilterAction(tableName),
new ChangeVersionsAction(tableName)
};
// Destructive actions to mess things around.
Action[] actions3 = new Action[] {
new MoveRegionsOfTableAction(moveRegionsSleepTime, moveRegionsMaxTime,
tableName),
new MoveRandomRegionOfTableAction(moveRandomRegionSleepTime, tableName),
new RestartRandomRsAction(restartRandomRSSleepTime),
new BatchRestartRsAction(batchRestartRSSleepTime, batchRestartRSRatio),
new RestartActiveMasterAction(restartActiveMasterSleepTime),
new RollingBatchRestartRsAction(rollingBatchRestartRSSleepTime,
rollingBatchRestartRSRatio),
new RestartRsHoldingMetaAction(restartRsHoldingMetaSleepTime)
};
// Action to log more info for debugging
Action[] actions4 = new Action[] {
new DumpClusterStatusAction()
};
return new PolicyBasedChaosMonkey(util,
new PeriodicRandomActionPolicy(action1Period, actions1),
new PeriodicRandomActionPolicy(action2Period, actions2),
new CompositeSequentialPolicy(
new DoActionsOncePolicy(action3Period, actions3),
new PeriodicRandomActionPolicy(action3Period, actions3)),
new PeriodicRandomActionPolicy(action4Period, actions4));
}
private void loadProperties() {
action1Period = Long.parseLong(this.properties.getProperty(
MonkeyConstants.PERIODIC_ACTION1_PERIOD,
MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD + ""));
action2Period = Long.parseLong(this.properties.getProperty(
MonkeyConstants.PERIODIC_ACTION2_PERIOD,
MonkeyConstants.DEFAULT_PERIODIC_ACTION2_PERIOD + ""));
action3Period = Long.parseLong(this.properties.getProperty(
MonkeyConstants.COMPOSITE_ACTION3_PERIOD,
MonkeyConstants.DEFAULT_COMPOSITE_ACTION3_PERIOD + ""));
action4Period = Long.parseLong(this.properties.getProperty(
MonkeyConstants.PERIODIC_ACTION4_PERIOD,
MonkeyConstants.DEFAULT_PERIODIC_ACTION4_PERIOD + ""));
moveRegionsMaxTime = Long.parseLong(this.properties.getProperty(
MonkeyConstants.MOVE_REGIONS_MAX_TIME,
MonkeyConstants.DEFAULT_MOVE_REGIONS_MAX_TIME + ""));
moveRegionsSleepTime = Long.parseLong(this.properties.getProperty(
MonkeyConstants.MOVE_REGIONS_SLEEP_TIME,
MonkeyConstants.DEFAULT_MOVE_REGIONS_SLEEP_TIME + ""));
moveRandomRegionSleepTime = Long.parseLong(this.properties.getProperty(
MonkeyConstants.MOVE_RANDOM_REGION_SLEEP_TIME,
MonkeyConstants.DEFAULT_MOVE_RANDOM_REGION_SLEEP_TIME + ""));
restartRandomRSSleepTime = Long.parseLong(this.properties.getProperty(
MonkeyConstants.RESTART_RANDOM_RS_SLEEP_TIME,
MonkeyConstants.DEFAULT_RESTART_RANDOM_RS_SLEEP_TIME + ""));
batchRestartRSSleepTime = Long.parseLong(this.properties.getProperty(
MonkeyConstants.BATCH_RESTART_RS_SLEEP_TIME,
MonkeyConstants.DEFAULT_BATCH_RESTART_RS_SLEEP_TIME + ""));
restartActiveMasterSleepTime = Long.parseLong(this.properties.getProperty(
MonkeyConstants.RESTART_ACTIVE_MASTER_SLEEP_TIME,
MonkeyConstants.DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME + ""));
rollingBatchRestartRSSleepTime = Long.parseLong(this.properties.getProperty(
MonkeyConstants.ROLLING_BATCH_RESTART_RS_SLEEP_TIME,
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME + ""));
rollingBatchRestartRSRatio = Float.parseFloat(this.properties.getProperty(
MonkeyConstants.ROLLING_BATCH_RESTART_RS_RATIO,
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO + ""));
restartRsHoldingMetaSleepTime = Long.parseLong(this.properties.getProperty(
MonkeyConstants.RESTART_RS_HOLDING_META_SLEEP_TIME,
MonkeyConstants.DEFAULT_RESTART_RS_HOLDING_META_SLEEP_TIME + ""));
compactTableRatio = Float.parseFloat(this.properties.getProperty(
MonkeyConstants.COMPACT_TABLE_ACTION_RATIO,
MonkeyConstants.DEFAULT_COMPACT_TABLE_ACTION_RATIO + ""));
compactRandomRegionRatio = Float.parseFloat(this.properties.getProperty(
MonkeyConstants.COMPACT_RANDOM_REGION_RATIO,
MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO + ""));
}
}

View File

@ -69,6 +69,8 @@ public abstract class MonkeyFactory {
public static final String SERVER_KILLING = "serverKilling"; public static final String SERVER_KILLING = "serverKilling";
public static final String STRESS_AM = "stressAM"; public static final String STRESS_AM = "stressAM";
public static final String NO_KILL = "noKill"; public static final String NO_KILL = "noKill";
public static final String MOB_NO_KILL = "mobNoKill";
public static final String MOB_SLOW_DETERMINISTIC = "mobSlowDeterministic";
public static Map<String, MonkeyFactory> FACTORIES = ImmutableMap.<String,MonkeyFactory>builder() public static Map<String, MonkeyFactory> FACTORIES = ImmutableMap.<String,MonkeyFactory>builder()
.put(CALM, new CalmMonkeyFactory()) .put(CALM, new CalmMonkeyFactory())
@ -77,6 +79,8 @@ public abstract class MonkeyFactory {
.put(SERVER_KILLING, new ServerKillingMonkeyFactory()) .put(SERVER_KILLING, new ServerKillingMonkeyFactory())
.put(STRESS_AM, new StressAssignmentManagerMonkeyFactory()) .put(STRESS_AM, new StressAssignmentManagerMonkeyFactory())
.put(NO_KILL, new NoKillMonkeyFactory()) .put(NO_KILL, new NoKillMonkeyFactory())
.put(MOB_NO_KILL, new MobNoKillMonkeyFactory())
.put(MOB_SLOW_DETERMINISTIC, new MobNoKillMonkeyFactory())
.build(); .build();
public static MonkeyFactory getFactory(String factoryName) { public static MonkeyFactory getFactory(String factoryName) {