HBASE-13157 Add Mob compaction actions and monkeys to Chaos Monkey
This commit is contained in:
Jonathan M Hsieh
Jonathan M Hsieh
parent
45711ebaaf
commit
0d6cac9b1e
|
|
@ -0,0 +1,65 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.chaos.actions;
|
||||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.Admin;
|
||||
|
||||
/**
|
||||
* Action that queues a table compaction.
|
||||
*/
|
||||
public class CompactMobAction extends Action {
|
||||
private final TableName tableName;
|
||||
private final int majorRatio;
|
||||
private final long sleepTime;
|
||||
|
||||
public CompactMobAction(TableName tableName, float majorRatio) {
|
||||
this(-1, tableName, majorRatio);
|
||||
}
|
||||
|
||||
public CompactMobAction(
|
||||
int sleepTime, TableName tableName, float majorRatio) {
|
||||
this.tableName = tableName;
|
||||
this.majorRatio = (int) (100 * majorRatio);
|
||||
this.sleepTime = sleepTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void perform() throws Exception {
|
||||
HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
|
||||
Admin admin = util.getHBaseAdmin();
|
||||
boolean major = RandomUtils.nextInt(100) < majorRatio;
|
||||
|
||||
LOG.info("Performing action: Compact mob of table " + tableName + ", major=" + major);
|
||||
try {
|
||||
if (major) {
|
||||
admin.majorCompactMob(tableName);
|
||||
} else {
|
||||
admin.compactMob(tableName);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
LOG.warn("Mob Compaction failed, might be caused by other chaos: " + ex.getMessage());
|
||||
}
|
||||
if (sleepTime > 0) {
|
||||
Thread.sleep(sleepTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.chaos.factories;
|
||||
|
||||
import org.apache.hadoop.hbase.chaos.actions.*;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
|
||||
import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
|
||||
import org.apache.hadoop.hbase.chaos.policies.TwoConcurrentActionPolicy;
|
||||
|
||||
/**
|
||||
* This is a copy of NoKillMonkeyFactory that also does mob compactions.
|
||||
*/
|
||||
public class MobNoKillMonkeyFactory extends MonkeyFactory {
|
||||
@Override public ChaosMonkey build() {
|
||||
Action[] actions1 = new Action[] {
|
||||
new CompactMobAction(tableName, MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD),
|
||||
new CompactTableAction(tableName, MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD),
|
||||
new CompactRandomRegionOfTableAction(tableName,
|
||||
MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO),
|
||||
new FlushTableAction(tableName),
|
||||
new FlushRandomRegionOfTableAction(tableName),
|
||||
new MoveRandomRegionOfTableAction(tableName)
|
||||
};
|
||||
|
||||
Action[] actions2 = new Action[] {
|
||||
new SplitRandomRegionOfTableAction(tableName),
|
||||
new MergeRandomAdjacentRegionsOfTableAction(tableName),
|
||||
new SnapshotTableAction(tableName),
|
||||
new AddColumnAction(tableName),
|
||||
new RemoveColumnAction(tableName, columnFamilies),
|
||||
new ChangeEncodingAction(tableName),
|
||||
new ChangeCompressionAction(tableName),
|
||||
new ChangeBloomFilterAction(tableName),
|
||||
new ChangeVersionsAction(tableName)
|
||||
};
|
||||
|
||||
Action[] actions3 = new Action[] {
|
||||
new MoveRegionsOfTableAction(MonkeyConstants.DEFAULT_MOVE_REGIONS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_MOVE_REGIONS_MAX_TIME,
|
||||
tableName),
|
||||
new MoveRandomRegionOfTableAction(MonkeyConstants.DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME,
|
||||
tableName),
|
||||
};
|
||||
|
||||
Action[] actions4 = new Action[] {
|
||||
new DumpClusterStatusAction()
|
||||
};
|
||||
|
||||
return new PolicyBasedChaosMonkey(util,
|
||||
new TwoConcurrentActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD, actions1, actions2),
|
||||
new PeriodicRandomActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION2_PERIOD,actions3),
|
||||
new PeriodicRandomActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION4_PERIOD,actions4));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.chaos.factories;
|
||||
|
||||
import org.apache.hadoop.hbase.chaos.actions.*;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
|
||||
import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy;
|
||||
import org.apache.hadoop.hbase.chaos.policies.DoActionsOncePolicy;
|
||||
import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
|
||||
|
||||
/**
|
||||
* This is a copy of SlowDeterministicMonkeyFactory that also does mob compactions.
|
||||
*/
|
||||
public class MobSlowDeterministicMonkeyFactory extends MonkeyFactory {
|
||||
|
||||
private long action1Period;
|
||||
private long action2Period;
|
||||
private long action3Period;
|
||||
private long action4Period;
|
||||
private long moveRegionsMaxTime;
|
||||
private long moveRegionsSleepTime;
|
||||
private long moveRandomRegionSleepTime;
|
||||
private long restartRandomRSSleepTime;
|
||||
private long batchRestartRSSleepTime;
|
||||
private float batchRestartRSRatio;
|
||||
private long restartActiveMasterSleepTime;
|
||||
private long rollingBatchRestartRSSleepTime;
|
||||
private float rollingBatchRestartRSRatio;
|
||||
private long restartRsHoldingMetaSleepTime;
|
||||
private float compactTableRatio;
|
||||
private float compactRandomRegionRatio;
|
||||
|
||||
@Override
|
||||
public ChaosMonkey build() {
|
||||
|
||||
loadProperties();
|
||||
// Actions such as compact/flush a table/region,
|
||||
// move one region around. They are not so destructive,
|
||||
// can be executed more frequently.
|
||||
Action[] actions1 = new Action[] {
|
||||
new CompactMobAction(tableName, compactTableRatio),
|
||||
new CompactTableAction(tableName, compactTableRatio),
|
||||
new CompactRandomRegionOfTableAction(tableName, compactRandomRegionRatio),
|
||||
new FlushTableAction(tableName),
|
||||
new FlushRandomRegionOfTableAction(tableName),
|
||||
new MoveRandomRegionOfTableAction(tableName)
|
||||
};
|
||||
|
||||
// Actions such as split/merge/snapshot.
|
||||
// They should not cause data loss, or unreliability
|
||||
// such as region stuck in transition.
|
||||
Action[] actions2 = new Action[] {
|
||||
new SplitRandomRegionOfTableAction(tableName),
|
||||
new MergeRandomAdjacentRegionsOfTableAction(tableName),
|
||||
new SnapshotTableAction(tableName),
|
||||
new AddColumnAction(tableName),
|
||||
new RemoveColumnAction(tableName, columnFamilies),
|
||||
new ChangeEncodingAction(tableName),
|
||||
new ChangeCompressionAction(tableName),
|
||||
new ChangeBloomFilterAction(tableName),
|
||||
new ChangeVersionsAction(tableName)
|
||||
};
|
||||
|
||||
// Destructive actions to mess things around.
|
||||
Action[] actions3 = new Action[] {
|
||||
new MoveRegionsOfTableAction(moveRegionsSleepTime, moveRegionsMaxTime,
|
||||
tableName),
|
||||
new MoveRandomRegionOfTableAction(moveRandomRegionSleepTime, tableName),
|
||||
new RestartRandomRsAction(restartRandomRSSleepTime),
|
||||
new BatchRestartRsAction(batchRestartRSSleepTime, batchRestartRSRatio),
|
||||
new RestartActiveMasterAction(restartActiveMasterSleepTime),
|
||||
new RollingBatchRestartRsAction(rollingBatchRestartRSSleepTime,
|
||||
rollingBatchRestartRSRatio),
|
||||
new RestartRsHoldingMetaAction(restartRsHoldingMetaSleepTime)
|
||||
};
|
||||
|
||||
// Action to log more info for debugging
|
||||
Action[] actions4 = new Action[] {
|
||||
new DumpClusterStatusAction()
|
||||
};
|
||||
|
||||
return new PolicyBasedChaosMonkey(util,
|
||||
new PeriodicRandomActionPolicy(action1Period, actions1),
|
||||
new PeriodicRandomActionPolicy(action2Period, actions2),
|
||||
new CompositeSequentialPolicy(
|
||||
new DoActionsOncePolicy(action3Period, actions3),
|
||||
new PeriodicRandomActionPolicy(action3Period, actions3)),
|
||||
new PeriodicRandomActionPolicy(action4Period, actions4));
|
||||
}
|
||||
|
||||
private void loadProperties() {
|
||||
|
||||
action1Period = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.PERIODIC_ACTION1_PERIOD,
|
||||
MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD + ""));
|
||||
action2Period = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.PERIODIC_ACTION2_PERIOD,
|
||||
MonkeyConstants.DEFAULT_PERIODIC_ACTION2_PERIOD + ""));
|
||||
action3Period = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.COMPOSITE_ACTION3_PERIOD,
|
||||
MonkeyConstants.DEFAULT_COMPOSITE_ACTION3_PERIOD + ""));
|
||||
action4Period = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.PERIODIC_ACTION4_PERIOD,
|
||||
MonkeyConstants.DEFAULT_PERIODIC_ACTION4_PERIOD + ""));
|
||||
moveRegionsMaxTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.MOVE_REGIONS_MAX_TIME,
|
||||
MonkeyConstants.DEFAULT_MOVE_REGIONS_MAX_TIME + ""));
|
||||
moveRegionsSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.MOVE_REGIONS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_MOVE_REGIONS_SLEEP_TIME + ""));
|
||||
moveRandomRegionSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.MOVE_RANDOM_REGION_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_MOVE_RANDOM_REGION_SLEEP_TIME + ""));
|
||||
restartRandomRSSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.RESTART_RANDOM_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_RESTART_RANDOM_RS_SLEEP_TIME + ""));
|
||||
batchRestartRSSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.BATCH_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_BATCH_RESTART_RS_SLEEP_TIME + ""));
|
||||
restartActiveMasterSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.RESTART_ACTIVE_MASTER_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME + ""));
|
||||
rollingBatchRestartRSSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchRestartRSRatio = Float.parseFloat(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_RATIO,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO + ""));
|
||||
restartRsHoldingMetaSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.RESTART_RS_HOLDING_META_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_RESTART_RS_HOLDING_META_SLEEP_TIME + ""));
|
||||
compactTableRatio = Float.parseFloat(this.properties.getProperty(
|
||||
MonkeyConstants.COMPACT_TABLE_ACTION_RATIO,
|
||||
MonkeyConstants.DEFAULT_COMPACT_TABLE_ACTION_RATIO + ""));
|
||||
compactRandomRegionRatio = Float.parseFloat(this.properties.getProperty(
|
||||
MonkeyConstants.COMPACT_RANDOM_REGION_RATIO,
|
||||
MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO + ""));
|
||||
}
|
||||
}
|
||||
|
|
@ -69,6 +69,8 @@ public abstract class MonkeyFactory {
|
|||
public static final String SERVER_KILLING = "serverKilling";
|
||||
public static final String STRESS_AM = "stressAM";
|
||||
public static final String NO_KILL = "noKill";
|
||||
public static final String MOB_NO_KILL = "mobNoKill";
|
||||
public static final String MOB_SLOW_DETERMINISTIC = "mobSlowDeterministic";
|
||||
|
||||
public static Map<String, MonkeyFactory> FACTORIES = ImmutableMap.<String,MonkeyFactory>builder()
|
||||
.put(CALM, new CalmMonkeyFactory())
|
||||
|
|
@ -77,6 +79,8 @@ public abstract class MonkeyFactory {
|
|||
.put(SERVER_KILLING, new ServerKillingMonkeyFactory())
|
||||
.put(STRESS_AM, new StressAssignmentManagerMonkeyFactory())
|
||||
.put(NO_KILL, new NoKillMonkeyFactory())
|
||||
.put(MOB_NO_KILL, new MobNoKillMonkeyFactory())
|
||||
.put(MOB_SLOW_DETERMINISTIC, new MobNoKillMonkeyFactory())
|
||||
.build();
|
||||
|
||||
public static MonkeyFactory getFactory(String factoryName) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue