HBASE-22982: region server suspend/resume and graceful rolling restart actions (#592)
* Add chaos monkey action for suspend/resume region servers * Add chaos monkey action for graceful rolling restart * Add these to relevant chaos monkeys Signed-off-by: Balazs Meszaros <meszibalu@apache.org> Signed-off-by: Peter Somogyi <psomogyi@apache.org>
This commit is contained in:
BukrosSzabolcs
Balazs Meszaros
parent
ea24ea7dd5
commit
f0dddd1cc2
|
|
@ -97,13 +97,13 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
|
||||
@Override
|
||||
public void startRegionServer(String hostname, int port) throws IOException {
|
||||
LOG.info("Starting RS on: " + hostname);
|
||||
LOG.info("Starting RS on: {}", hostname);
|
||||
clusterManager.start(ServiceType.HBASE_REGIONSERVER, hostname, port);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void killRegionServer(ServerName serverName) throws IOException {
|
||||
LOG.info("Aborting RS: " + serverName.getServerName());
|
||||
LOG.info("Aborting RS: {}", serverName.getServerName());
|
||||
killedRegionServers.add(serverName);
|
||||
clusterManager.kill(ServiceType.HBASE_REGIONSERVER,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
|
|
@ -116,7 +116,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
|
||||
@Override
|
||||
public void stopRegionServer(ServerName serverName) throws IOException {
|
||||
LOG.info("Stopping RS: " + serverName.getServerName());
|
||||
LOG.info("Stopping RS: {}", serverName.getServerName());
|
||||
clusterManager.stop(ServiceType.HBASE_REGIONSERVER,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
|
@ -126,22 +126,36 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
waitForServiceToStop(ServiceType.HBASE_REGIONSERVER, serverName, timeout);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void suspendRegionServer(ServerName serverName) throws IOException {
|
||||
LOG.info("Suspend RS: {}", serverName.getServerName());
|
||||
clusterManager.suspend(ServiceType.HBASE_REGIONSERVER,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void resumeRegionServer(ServerName serverName) throws IOException {
|
||||
LOG.info("Resume RS: {}", serverName.getServerName());
|
||||
clusterManager.resume(ServiceType.HBASE_REGIONSERVER,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startZkNode(String hostname, int port) throws IOException {
|
||||
LOG.info("Starting ZooKeeper node on: " + hostname);
|
||||
LOG.info("Starting ZooKeeper node on: {}", hostname);
|
||||
clusterManager.start(ServiceType.ZOOKEEPER_SERVER, hostname, port);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void killZkNode(ServerName serverName) throws IOException {
|
||||
LOG.info("Aborting ZooKeeper node on: " + serverName.getServerName());
|
||||
LOG.info("Aborting ZooKeeper node on: {}", serverName.getServerName());
|
||||
clusterManager.kill(ServiceType.ZOOKEEPER_SERVER,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopZkNode(ServerName serverName) throws IOException {
|
||||
LOG.info("Stopping ZooKeeper node: " + serverName.getServerName());
|
||||
LOG.info("Stopping ZooKeeper node: {}", serverName.getServerName());
|
||||
clusterManager.stop(ServiceType.ZOOKEEPER_SERVER,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
|
@ -158,21 +172,21 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
|
||||
@Override
|
||||
public void startDataNode(ServerName serverName) throws IOException {
|
||||
LOG.info("Starting data node on: " + serverName.getServerName());
|
||||
LOG.info("Starting data node on: {}", serverName.getServerName());
|
||||
clusterManager.start(ServiceType.HADOOP_DATANODE,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void killDataNode(ServerName serverName) throws IOException {
|
||||
LOG.info("Aborting data node on: " + serverName.getServerName());
|
||||
LOG.info("Aborting data node on: {}", serverName.getServerName());
|
||||
clusterManager.kill(ServiceType.HADOOP_DATANODE,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopDataNode(ServerName serverName) throws IOException {
|
||||
LOG.info("Stopping data node on: " + serverName.getServerName());
|
||||
LOG.info("Stopping data node on: {}", serverName.getServerName());
|
||||
clusterManager.stop(ServiceType.HADOOP_DATANODE,
|
||||
serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
|
@ -189,21 +203,21 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
|
||||
@Override
|
||||
public void startNameNode(ServerName serverName) throws IOException {
|
||||
LOG.info("Starting name node on: " + serverName.getServerName());
|
||||
LOG.info("Starting name node on: {}", serverName.getServerName());
|
||||
clusterManager.start(ServiceType.HADOOP_NAMENODE, serverName.getHostname(),
|
||||
serverName.getPort());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void killNameNode(ServerName serverName) throws IOException {
|
||||
LOG.info("Aborting name node on: " + serverName.getServerName());
|
||||
LOG.info("Aborting name node on: {}", serverName.getServerName());
|
||||
clusterManager.kill(ServiceType.HADOOP_NAMENODE, serverName.getHostname(),
|
||||
serverName.getPort());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopNameNode(ServerName serverName) throws IOException {
|
||||
LOG.info("Stopping name node on: " + serverName.getServerName());
|
||||
LOG.info("Stopping name node on: {}", serverName.getServerName());
|
||||
clusterManager.stop(ServiceType.HADOOP_NAMENODE, serverName.getHostname(),
|
||||
serverName.getPort());
|
||||
}
|
||||
|
|
@ -220,7 +234,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
|
||||
private void waitForServiceToStop(ServiceType service, ServerName serverName, long timeout)
|
||||
throws IOException {
|
||||
LOG.info("Waiting for service: " + service + " to stop: " + serverName.getServerName());
|
||||
LOG.info("Waiting for service: {} to stop: {}", service, serverName.getServerName());
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
while ((System.currentTimeMillis() - start) < timeout) {
|
||||
|
|
@ -234,7 +248,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
|
||||
private void waitForServiceToStart(ServiceType service, ServerName serverName, long timeout)
|
||||
throws IOException {
|
||||
LOG.info("Waiting for service: " + service + " to start: " + serverName.getServerName());
|
||||
LOG.info("Waiting for service: {} to start: ", service, serverName.getServerName());
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
while ((System.currentTimeMillis() - start) < timeout) {
|
||||
|
|
@ -248,19 +262,19 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
|
||||
@Override
|
||||
public void startMaster(String hostname, int port) throws IOException {
|
||||
LOG.info("Starting Master on: " + hostname + ":" + port);
|
||||
LOG.info("Starting Master on: {}:{}", hostname, port);
|
||||
clusterManager.start(ServiceType.HBASE_MASTER, hostname, port);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void killMaster(ServerName serverName) throws IOException {
|
||||
LOG.info("Aborting Master: " + serverName.getServerName());
|
||||
LOG.info("Aborting Master: {}", serverName.getServerName());
|
||||
clusterManager.kill(ServiceType.HBASE_MASTER, serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopMaster(ServerName serverName) throws IOException {
|
||||
LOG.info("Stopping Master: " + serverName.getServerName());
|
||||
LOG.info("Stopping Master: {}", serverName.getServerName());
|
||||
clusterManager.stop(ServiceType.HBASE_MASTER, serverName.getHostname(), serverName.getPort());
|
||||
}
|
||||
|
||||
|
|
@ -294,7 +308,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
regionLoc = locator.getRegionLocation(startKey, true);
|
||||
}
|
||||
if (regionLoc == null) {
|
||||
LOG.warn("Cannot find region server holding region " + Bytes.toStringBinary(regionName));
|
||||
LOG.warn("Cannot find region server holding region {}", Bytes.toStringBinary(regionName));
|
||||
return null;
|
||||
}
|
||||
return regionLoc.getServerName();
|
||||
|
|
@ -338,15 +352,15 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
//check whether current master has changed
|
||||
final ServerName initMaster = initial.getMasterName();
|
||||
if (!ServerName.isSameAddress(initMaster, current.getMasterName())) {
|
||||
LOG.info("Restoring cluster - Initial active master : " + initMaster.getAddress() +
|
||||
" has changed to : " + current.getMasterName().getAddress());
|
||||
LOG.info("Restoring cluster - Initial active master : {} has changed to : {}",
|
||||
initMaster.getAddress(), current.getMasterName().getAddress());
|
||||
// If initial master is stopped, start it, before restoring the state.
|
||||
// It will come up as a backup master, if there is already an active master.
|
||||
try {
|
||||
if (!clusterManager.isRunning(ServiceType.HBASE_MASTER,
|
||||
initMaster.getHostname(), initMaster.getPort())) {
|
||||
LOG.info("Restoring cluster - starting initial active master at:"
|
||||
+ initMaster.getAddress());
|
||||
LOG.info("Restoring cluster - starting initial active master at:{}",
|
||||
initMaster.getAddress());
|
||||
startMaster(initMaster.getHostname(), initMaster.getPort());
|
||||
}
|
||||
|
||||
|
|
@ -356,11 +370,11 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
// 3. Start backup masters
|
||||
for (ServerName currentBackup : current.getBackupMasterNames()) {
|
||||
if (!ServerName.isSameAddress(currentBackup, initMaster)) {
|
||||
LOG.info("Restoring cluster - stopping backup master: " + currentBackup);
|
||||
LOG.info("Restoring cluster - stopping backup master: {}", currentBackup);
|
||||
stopMaster(currentBackup);
|
||||
}
|
||||
}
|
||||
LOG.info("Restoring cluster - stopping active master: " + current.getMasterName());
|
||||
LOG.info("Restoring cluster - stopping active master: {}", current.getMasterName());
|
||||
stopMaster(current.getMasterName());
|
||||
waitForActiveAndReadyMaster(); // wait so that active master takes over
|
||||
} catch (IOException ex) {
|
||||
|
|
@ -376,8 +390,8 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
if (!clusterManager.isRunning(ServiceType.HBASE_MASTER,
|
||||
backup.getHostname(),
|
||||
backup.getPort())) {
|
||||
LOG.info("Restoring cluster - starting initial backup master: "
|
||||
+ backup.getAddress());
|
||||
LOG.info("Restoring cluster - starting initial backup master: {}",
|
||||
backup.getAddress());
|
||||
startMaster(backup.getHostname(), backup.getPort());
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
|
|
@ -401,7 +415,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
for (ServerName sn:toStart) {
|
||||
try {
|
||||
if(!clusterManager.isRunning(ServiceType.HBASE_MASTER, sn.getHostname(), sn.getPort())) {
|
||||
LOG.info("Restoring cluster - starting initial backup master: " + sn.getAddress());
|
||||
LOG.info("Restoring cluster - starting initial backup master: {}", sn.getAddress());
|
||||
startMaster(sn.getHostname(), sn.getPort());
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
|
|
@ -412,7 +426,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
for (ServerName sn:toKill) {
|
||||
try {
|
||||
if(clusterManager.isRunning(ServiceType.HBASE_MASTER, sn.getHostname(), sn.getPort())) {
|
||||
LOG.info("Restoring cluster - stopping backup master: " + sn.getAddress());
|
||||
LOG.info("Restoring cluster - stopping backup master: {}", sn.getAddress());
|
||||
stopMaster(sn);
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
|
|
@ -421,8 +435,8 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
}
|
||||
}
|
||||
if (!deferred.isEmpty()) {
|
||||
LOG.warn("Restoring cluster - restoring region servers reported "
|
||||
+ deferred.size() + " errors:");
|
||||
LOG.warn("Restoring cluster - restoring region servers reported {} errors:",
|
||||
deferred.size());
|
||||
for (int i=0; i<deferred.size() && i < 3; i++) {
|
||||
LOG.warn(Objects.toString(deferred.get(i)));
|
||||
}
|
||||
|
|
@ -464,7 +478,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
try {
|
||||
if (!clusterManager.isRunning(ServiceType.HBASE_REGIONSERVER, sn.getHostname(),
|
||||
sn.getPort()) && master.getPort() != sn.getPort()) {
|
||||
LOG.info("Restoring cluster - starting initial region server: " + sn.getAddress());
|
||||
LOG.info("Restoring cluster - starting initial region server: {}", sn.getAddress());
|
||||
startRegionServer(sn.getHostname(), sn.getPort());
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
|
|
@ -476,7 +490,7 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
try {
|
||||
if (clusterManager.isRunning(ServiceType.HBASE_REGIONSERVER, sn.getHostname(),
|
||||
sn.getPort()) && master.getPort() != sn.getPort()) {
|
||||
LOG.info("Restoring cluster - stopping initial region server: " + sn.getAddress());
|
||||
LOG.info("Restoring cluster - stopping initial region server: {}", sn.getAddress());
|
||||
stopRegionServer(sn);
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
|
|
@ -484,8 +498,8 @@ public class DistributedHBaseCluster extends HBaseCluster {
|
|||
}
|
||||
}
|
||||
if (!deferred.isEmpty()) {
|
||||
LOG.warn("Restoring cluster - restoring region servers reported "
|
||||
+ deferred.size() + " errors:");
|
||||
LOG.warn("Restoring cluster - restoring region servers reported {} errors:",
|
||||
deferred.size());
|
||||
for (int i=0; i<deferred.size() && i < 3; i++) {
|
||||
LOG.warn(Objects.toString(deferred.get(i)));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HBaseCluster;
|
|||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HRegionInfo;
|
||||
import org.apache.hadoop.hbase.IntegrationTestingUtility;
|
||||
import org.apache.hadoop.hbase.MiniHBaseCluster;
|
||||
import org.apache.hadoop.hbase.ServerMetrics;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
|
|
@ -150,78 +151,106 @@ public class Action {
|
|||
}
|
||||
|
||||
protected void killMaster(ServerName server) throws IOException {
|
||||
LOG.info("Killing master " + server);
|
||||
LOG.info("Killing master {}", server);
|
||||
cluster.killMaster(server);
|
||||
cluster.waitForMasterToStop(server, killMasterTimeout);
|
||||
LOG.info("Killed master " + server);
|
||||
}
|
||||
|
||||
protected void startMaster(ServerName server) throws IOException {
|
||||
LOG.info("Starting master " + server.getHostname());
|
||||
LOG.info("Starting master {}", server.getHostname());
|
||||
cluster.startMaster(server.getHostname(), server.getPort());
|
||||
cluster.waitForActiveAndReadyMaster(startMasterTimeout);
|
||||
LOG.info("Started master " + server.getHostname());
|
||||
}
|
||||
|
||||
protected void stopRs(ServerName server) throws IOException {
|
||||
LOG.info("Stopping regionserver {}", server);
|
||||
cluster.stopRegionServer(server);
|
||||
cluster.waitForRegionServerToStop(server, killRsTimeout);
|
||||
LOG.info("Stoppiong regionserver {}. Reported num of rs:{}", server,
|
||||
cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
}
|
||||
|
||||
protected void suspendRs(ServerName server) throws IOException {
|
||||
LOG.info("Suspending regionserver {}", server);
|
||||
cluster.suspendRegionServer(server);
|
||||
if(!(cluster instanceof MiniHBaseCluster)){
|
||||
cluster.waitForRegionServerToStop(server, killRsTimeout);
|
||||
}
|
||||
LOG.info("Suspending regionserver {}. Reported num of rs:{}", server,
|
||||
cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
}
|
||||
|
||||
protected void resumeRs(ServerName server) throws IOException {
|
||||
LOG.info("Resuming regionserver {}", server);
|
||||
cluster.resumeRegionServer(server);
|
||||
if(!(cluster instanceof MiniHBaseCluster)){
|
||||
cluster.waitForRegionServerToStart(server.getHostname(), server.getPort(), startRsTimeout);
|
||||
}
|
||||
LOG.info("Resuming regionserver {}. Reported num of rs:{}", server,
|
||||
cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
}
|
||||
|
||||
protected void killRs(ServerName server) throws IOException {
|
||||
LOG.info("Killing regionserver " + server);
|
||||
LOG.info("Killing regionserver {}", server);
|
||||
cluster.killRegionServer(server);
|
||||
cluster.waitForRegionServerToStop(server, killRsTimeout);
|
||||
LOG.info("Killed regionserver " + server + ". Reported num of rs:"
|
||||
+ cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
LOG.info("Killed regionserver {}. Reported num of rs:{}", server,
|
||||
cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
}
|
||||
|
||||
protected void startRs(ServerName server) throws IOException {
|
||||
LOG.info("Starting regionserver " + server.getAddress());
|
||||
LOG.info("Starting regionserver {}", server.getAddress());
|
||||
cluster.startRegionServer(server.getHostname(), server.getPort());
|
||||
cluster.waitForRegionServerToStart(server.getHostname(), server.getPort(), startRsTimeout);
|
||||
LOG.info("Started regionserver " + server.getAddress() + ". Reported num of rs:"
|
||||
+ cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
LOG.info("Started regionserver {}. Reported num of rs:{}", server.getAddress(),
|
||||
cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
}
|
||||
|
||||
protected void killZKNode(ServerName server) throws IOException {
|
||||
LOG.info("Killing zookeeper node " + server);
|
||||
LOG.info("Killing zookeeper node {}", server);
|
||||
cluster.killZkNode(server);
|
||||
cluster.waitForZkNodeToStop(server, killZkNodeTimeout);
|
||||
LOG.info("Killed zookeeper node " + server + ". Reported num of rs:"
|
||||
+ cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
LOG.info("Killed zookeeper node {}. Reported num of rs:{}", server,
|
||||
cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
}
|
||||
|
||||
protected void startZKNode(ServerName server) throws IOException {
|
||||
LOG.info("Starting zookeeper node " + server.getHostname());
|
||||
LOG.info("Starting zookeeper node {}", server.getHostname());
|
||||
cluster.startZkNode(server.getHostname(), server.getPort());
|
||||
cluster.waitForZkNodeToStart(server, startZkNodeTimeout);
|
||||
LOG.info("Started zookeeper node " + server);
|
||||
LOG.info("Started zookeeper node {}", server);
|
||||
}
|
||||
|
||||
protected void killDataNode(ServerName server) throws IOException {
|
||||
LOG.info("Killing datanode " + server);
|
||||
LOG.info("Killing datanode {}", server);
|
||||
cluster.killDataNode(server);
|
||||
cluster.waitForDataNodeToStop(server, killDataNodeTimeout);
|
||||
LOG.info("Killed datanode " + server + ". Reported num of rs:"
|
||||
+ cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
LOG.info("Killed datanode {}. Reported num of rs:{}", server,
|
||||
cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
}
|
||||
|
||||
protected void startDataNode(ServerName server) throws IOException {
|
||||
LOG.info("Starting datanode " + server.getHostname());
|
||||
LOG.info("Starting datanode {}", server.getHostname());
|
||||
cluster.startDataNode(server);
|
||||
cluster.waitForDataNodeToStart(server, startDataNodeTimeout);
|
||||
LOG.info("Started datanode " + server);
|
||||
LOG.info("Started datanode {}", server);
|
||||
}
|
||||
|
||||
protected void killNameNode(ServerName server) throws IOException {
|
||||
LOG.info("Killing namenode :-" + server.getHostname());
|
||||
LOG.info("Killing namenode :-{}", server.getHostname());
|
||||
cluster.killNameNode(server);
|
||||
cluster.waitForNameNodeToStop(server, killNameNodeTimeout);
|
||||
LOG.info("Killed namenode:" + server + ". Reported num of rs:"
|
||||
+ cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
LOG.info("Killed namenode:{}. Reported num of rs:{}", server,
|
||||
cluster.getClusterMetrics().getLiveServerMetrics().size());
|
||||
}
|
||||
|
||||
protected void startNameNode(ServerName server) throws IOException {
|
||||
LOG.info("Starting Namenode :-" + server.getHostname());
|
||||
LOG.info("Starting Namenode :-{}", server.getHostname());
|
||||
cluster.startNameNode(server);
|
||||
cluster.waitForNameNodeToStart(server, startNameNodeTimeout);
|
||||
LOG.info("Started namenode:" + server);
|
||||
LOG.info("Started namenode:{}", server);
|
||||
}
|
||||
protected void unbalanceRegions(ClusterMetrics clusterStatus,
|
||||
List<ServerName> fromServers, List<ServerName> toServers,
|
||||
|
|
@ -234,7 +263,7 @@ public class Action {
|
|||
// Ugh.
|
||||
List<byte[]> regions = new LinkedList<>(serverLoad.getRegionMetrics().keySet());
|
||||
int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size());
|
||||
LOG.debug("Removing " + victimRegionCount + " regions from " + sn);
|
||||
LOG.debug("Removing {} regions from {}", victimRegionCount, sn);
|
||||
for (int i = 0; i < victimRegionCount; ++i) {
|
||||
int victimIx = RandomUtils.nextInt(0, regions.size());
|
||||
String regionId = HRegionInfo.encodeRegionName(regions.remove(victimIx));
|
||||
|
|
@ -242,8 +271,8 @@ public class Action {
|
|||
}
|
||||
}
|
||||
|
||||
LOG.info("Moving " + victimRegions.size() + " regions from " + fromServers.size()
|
||||
+ " servers to " + toServers.size() + " different servers");
|
||||
LOG.info("Moving {} regions from {} servers to {} different servers", victimRegions.size(),
|
||||
fromServers.size(), toServers.size());
|
||||
Admin admin = this.context.getHBaseIntegrationTestingUtility().getAdmin();
|
||||
for (byte[] victimRegion : victimRegions) {
|
||||
// Don't keep moving regions if we're
|
||||
|
|
@ -269,6 +298,15 @@ public class Action {
|
|||
}
|
||||
}
|
||||
|
||||
protected void setBalancer(boolean onOrOff, boolean synchronous) throws Exception {
|
||||
Admin admin = this.context.getHBaseIntegrationTestingUtility().getAdmin();
|
||||
try {
|
||||
admin.balancerSwitch(onOrOff, synchronous);
|
||||
} catch (Exception e) {
|
||||
LOG.warn("Got exception while switching balance ", e);
|
||||
}
|
||||
}
|
||||
|
||||
public Configuration getConf() {
|
||||
return cluster.getConf();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,73 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.chaos.actions;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import org.apache.commons.lang3.RandomUtils;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.util.RegionMover;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Gracefully restarts every regionserver in a rolling fashion. At each step, it unloads,
|
||||
* restarts the loads every rs server sleeping randomly (0-sleepTime) in between servers.
|
||||
*/
|
||||
public class GracefulRollingRestartRsAction extends RestartActionBaseAction {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(GracefulRollingRestartRsAction.class);
|
||||
|
||||
public GracefulRollingRestartRsAction(long sleepTime) {
|
||||
super(sleepTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void perform() throws Exception {
|
||||
LOG.info("Performing action: Rolling restarting non-master region servers");
|
||||
List<ServerName> selectedServers = selectServers();
|
||||
|
||||
LOG.info("Disabling balancer to make unloading possible");
|
||||
setBalancer(false, true);
|
||||
|
||||
for (ServerName server : selectedServers) {
|
||||
String rsName = server.getAddress().toString();
|
||||
try (RegionMover rm =
|
||||
new RegionMover.RegionMoverBuilder(rsName, getConf()).ack(true).build()) {
|
||||
LOG.info("Unloading {}", server);
|
||||
rm.unload();
|
||||
LOG.info("Restarting {}", server);
|
||||
gracefulRestartRs(server, sleepTime);
|
||||
LOG.info("Loading {}", server);
|
||||
rm.load();
|
||||
} catch (Shell.ExitCodeException e) {
|
||||
LOG.info("Problem restarting but presume successful; code={}", e.getExitCode(), e);
|
||||
}
|
||||
sleep(RandomUtils.nextInt(0, (int)sleepTime));
|
||||
}
|
||||
LOG.info("Enabling balancer");
|
||||
setBalancer(true, true);
|
||||
}
|
||||
|
||||
protected List<ServerName> selectServers() throws IOException {
|
||||
return Arrays.asList(getCurrentServers());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -45,19 +45,42 @@ public class RestartActionBaseAction extends Action {
|
|||
return;
|
||||
}
|
||||
|
||||
LOG.info("Killing master: {}", server);
|
||||
killMaster(server);
|
||||
sleep(sleepTime);
|
||||
LOG.info("Starting master: {}", server);
|
||||
startMaster(server);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop and then restart the region server instead of killing it.
|
||||
* @param server hostname to restart the regionserver on
|
||||
* @param sleepTime number of milliseconds between stop and restart
|
||||
* @throws IOException if something goes wrong
|
||||
*/
|
||||
void gracefulRestartRs(ServerName server, long sleepTime) throws IOException {
|
||||
sleepTime = Math.max(sleepTime, 1000);
|
||||
// Don't try the stop if we're stopping already
|
||||
if (context.isStopping()) {
|
||||
return;
|
||||
}
|
||||
LOG.info("Stopping region server: {}", server);
|
||||
stopRs(server);
|
||||
sleep(sleepTime);
|
||||
LOG.info("Starting region server: {}", server);
|
||||
startRs(server);
|
||||
}
|
||||
|
||||
void restartRs(ServerName server, long sleepTime) throws IOException {
|
||||
sleepTime = Math.max(sleepTime, 1000);
|
||||
// Don't try the kill if we're stopping
|
||||
if (context.isStopping()) {
|
||||
return;
|
||||
}
|
||||
LOG.info("Killing region server: {}", server);
|
||||
killRs(server);
|
||||
sleep(sleepTime);
|
||||
LOG.info("Starting region server: {}", server);
|
||||
startRs(server);
|
||||
}
|
||||
|
||||
|
|
@ -67,8 +90,10 @@ public class RestartActionBaseAction extends Action {
|
|||
if (context.isStopping()) {
|
||||
return;
|
||||
}
|
||||
LOG.info("Killing zookeeper node: {}", server);
|
||||
killZKNode(server);
|
||||
sleep(sleepTime);
|
||||
LOG.info("Starting zookeeper node: {}", server);
|
||||
startZKNode(server);
|
||||
}
|
||||
|
||||
|
|
@ -78,8 +103,10 @@ public class RestartActionBaseAction extends Action {
|
|||
if (context.isStopping()) {
|
||||
return;
|
||||
}
|
||||
LOG.info("Killing data node: {}", server);
|
||||
killDataNode(server);
|
||||
sleep(sleepTime);
|
||||
LOG.info("Starting data node: {}", server);
|
||||
startDataNode(server);
|
||||
}
|
||||
|
||||
|
|
@ -89,8 +116,10 @@ public class RestartActionBaseAction extends Action {
|
|||
if (context.isStopping()) {
|
||||
return;
|
||||
}
|
||||
LOG.info("Killing name node: {}", server);
|
||||
killNameNode(server);
|
||||
sleep(sleepTime);
|
||||
LOG.info("Starting name node: {}", server);
|
||||
startNameNode(server);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.chaos.actions;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Queue;
|
||||
|
||||
import org.apache.commons.lang3.RandomUtils;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
|
||||
import org.apache.hadoop.hbase.util.Threads;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Suspend then resume a ratio of the regionservers in a rolling fashion. At each step, either
|
||||
* suspend a server, or resume one, sleeping (sleepTime) in between steps. The parameter
|
||||
* maxSuspendedServers limits the maximum number of servers that can be down at the same time
|
||||
* during rolling restarts.
|
||||
*/
|
||||
public class RollingBatchSuspendResumeRsAction extends Action {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(RollingBatchSuspendResumeRsAction.class);
|
||||
private float ratio;
|
||||
private long sleepTime;
|
||||
private int maxSuspendedServers; // number of maximum suspended servers at any given time.
|
||||
|
||||
public RollingBatchSuspendResumeRsAction(long sleepTime, float ratio) {
|
||||
this(sleepTime, ratio, 5);
|
||||
}
|
||||
|
||||
public RollingBatchSuspendResumeRsAction(long sleepTime, float ratio, int maxSuspendedServers) {
|
||||
this.ratio = ratio;
|
||||
this.sleepTime = sleepTime;
|
||||
this.maxSuspendedServers = maxSuspendedServers;
|
||||
}
|
||||
|
||||
enum SuspendOrResume {
|
||||
SUSPEND, RESUME
|
||||
}
|
||||
|
||||
@Override
|
||||
public void perform() throws Exception {
|
||||
LOG.info(String.format("Performing action: Rolling batch restarting %d%% of region servers",
|
||||
(int) (ratio * 100)));
|
||||
List<ServerName> selectedServers = selectServers();
|
||||
|
||||
Queue<ServerName> serversToBeSuspended = new LinkedList<>(selectedServers);
|
||||
Queue<ServerName> suspendedServers = new LinkedList<>();
|
||||
|
||||
// loop while there are servers to be suspended or suspended servers to be resumed
|
||||
while ((!serversToBeSuspended.isEmpty() || !suspendedServers.isEmpty()) && !context
|
||||
.isStopping()) {
|
||||
SuspendOrResume action;
|
||||
|
||||
if (serversToBeSuspended.isEmpty()) { // no more servers to suspend
|
||||
action = SuspendOrResume.RESUME;
|
||||
} else if (suspendedServers.isEmpty()) {
|
||||
action = SuspendOrResume.SUSPEND; // no more servers to resume
|
||||
} else if (suspendedServers.size() >= maxSuspendedServers) {
|
||||
// we have too many suspended servers. Don't suspend any more
|
||||
action = SuspendOrResume.RESUME;
|
||||
} else {
|
||||
// do a coin toss
|
||||
action = RandomUtils.nextBoolean() ? SuspendOrResume.SUSPEND : SuspendOrResume.RESUME;
|
||||
}
|
||||
|
||||
ServerName server;
|
||||
switch (action) {
|
||||
case SUSPEND:
|
||||
server = serversToBeSuspended.remove();
|
||||
try {
|
||||
suspendRs(server);
|
||||
} catch (Shell.ExitCodeException e) {
|
||||
LOG.warn("Problem suspending but presume successful; code={}", e.getExitCode(), e);
|
||||
}
|
||||
suspendedServers.add(server);
|
||||
break;
|
||||
case RESUME:
|
||||
server = suspendedServers.remove();
|
||||
try {
|
||||
resumeRs(server);
|
||||
} catch (Shell.ExitCodeException e) {
|
||||
LOG.info("Problem resuming, will retry; code={}", e.getExitCode(), e);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
LOG.info("Sleeping for:{}", sleepTime);
|
||||
Threads.sleep(sleepTime);
|
||||
}
|
||||
}
|
||||
|
||||
protected List<ServerName> selectServers() throws IOException {
|
||||
return PolicyBasedChaosMonkey.selectRandomItems(getCurrentServers(), ratio);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -45,6 +45,9 @@ public interface MonkeyConstants {
|
|||
String UNBALANCE_WAIT_AFTER_BALANCE_MS = "unbalance.action.wait.after.period";
|
||||
String UNBALANCE_KILL_META_RS = "unbalance.action.kill.meta.rs";
|
||||
String DECREASE_HFILE_SIZE_SLEEP_TIME = "decrease.hfile.size.sleep.time";
|
||||
String GRACEFUL_RESTART_RS_SLEEP_TIME = "graceful.restart.rs.sleep.time";
|
||||
String ROLLING_BATCH_SUSPEND_RS_SLEEP_TIME = "rolling.batch.suspend.rs.sleep.time";
|
||||
String ROLLING_BATCH_SUSPEND_RS_RATIO = "rolling.batch.suspend.rs.ratio";
|
||||
|
||||
/**
|
||||
* A Set of prefixes which encompasses all of the configuration properties for the ChaosMonky.
|
||||
|
|
@ -75,4 +78,7 @@ public interface MonkeyConstants {
|
|||
long DEFAULT_UNBALANCE_WAIT_AFTER_BALANCE_MS = 5 * 1000;
|
||||
boolean DEFAULT_UNBALANCE_KILL_META_RS = true;
|
||||
long DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME = 30 * 1000;
|
||||
long DEFAULT_GRACEFUL_RESTART_RS_SLEEP_TIME = 5000;
|
||||
long DEFAULT_ROLLING_BATCH_SUSPEND_RS_SLEEP_TIME = 30 * 1000;
|
||||
float DEFAULT_ROLLING_BATCH_SUSPEND_RS_RATIO = 1.0f;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,11 +21,13 @@ package org.apache.hadoop.hbase.chaos.factories;
|
|||
import org.apache.hadoop.hbase.chaos.actions.Action;
|
||||
import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.ForceBalancerAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.GracefulRollingRestartRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RestartRandomDataNodeAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsExceptMetaAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RestartRandomZKNodeAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsExceptMetaAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RollingBatchSuspendResumeRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
|
||||
import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy;
|
||||
|
|
@ -38,8 +40,13 @@ import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
|
|||
*/
|
||||
public class ServerAndDependenciesKillingMonkeyFactory extends MonkeyFactory {
|
||||
|
||||
private long gracefulRollingRestartTSSLeepTime;
|
||||
private long rollingBatchSuspendRSSleepTime;
|
||||
private float rollingBatchSuspendtRSRatio;
|
||||
|
||||
@Override
|
||||
public ChaosMonkey build() {
|
||||
loadProperties();
|
||||
|
||||
// Destructive actions to mess things around. Cannot run batch restart.
|
||||
Action[] actions1 = new Action[]{
|
||||
|
|
@ -48,7 +55,10 @@ public class ServerAndDependenciesKillingMonkeyFactory extends MonkeyFactory {
|
|||
new RollingBatchRestartRsExceptMetaAction(5000, 1.0f, 2), // only allow 2 servers to be dead.
|
||||
new ForceBalancerAction(),
|
||||
new RestartRandomDataNodeAction(60000),
|
||||
new RestartRandomZKNodeAction(60000)
|
||||
new RestartRandomZKNodeAction(60000),
|
||||
new GracefulRollingRestartRsAction(gracefulRollingRestartTSSLeepTime),
|
||||
new RollingBatchSuspendResumeRsAction(rollingBatchSuspendRSSleepTime,
|
||||
rollingBatchSuspendtRSRatio)
|
||||
};
|
||||
|
||||
// Action to log more info for debugging
|
||||
|
|
@ -62,4 +72,16 @@ public class ServerAndDependenciesKillingMonkeyFactory extends MonkeyFactory {
|
|||
new PeriodicRandomActionPolicy(60 * 1000, actions1)),
|
||||
new PeriodicRandomActionPolicy(60 * 1000, actions2));
|
||||
}
|
||||
|
||||
private void loadProperties() {
|
||||
gracefulRollingRestartTSSLeepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.GRACEFUL_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_GRACEFUL_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchSuspendRSSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchSuspendtRSRatio = Float.parseFloat(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_RATIO,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO + ""));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,9 +21,11 @@ package org.apache.hadoop.hbase.chaos.factories;
|
|||
import org.apache.hadoop.hbase.chaos.actions.Action;
|
||||
import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.ForceBalancerAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.GracefulRollingRestartRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsExceptMetaAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsExceptMetaAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RollingBatchSuspendResumeRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
|
||||
import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy;
|
||||
|
|
@ -36,15 +38,23 @@ import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
|
|||
*/
|
||||
public class ServerKillingMonkeyFactory extends MonkeyFactory {
|
||||
|
||||
private long gracefulRollingRestartTSSLeepTime;
|
||||
private long rollingBatchSuspendRSSleepTime;
|
||||
private float rollingBatchSuspendtRSRatio;
|
||||
|
||||
@Override
|
||||
public ChaosMonkey build() {
|
||||
loadProperties();
|
||||
|
||||
// Destructive actions to mess things around. Cannot run batch restart
|
||||
Action[] actions1 = new Action[] {
|
||||
new RestartRandomRsExceptMetaAction(60000),
|
||||
new RestartActiveMasterAction(5000),
|
||||
new RollingBatchRestartRsExceptMetaAction(5000, 1.0f, 2), //only allow 2 servers to be dead
|
||||
new ForceBalancerAction()
|
||||
new ForceBalancerAction(),
|
||||
new GracefulRollingRestartRsAction(gracefulRollingRestartTSSLeepTime),
|
||||
new RollingBatchSuspendResumeRsAction(rollingBatchSuspendRSSleepTime,
|
||||
rollingBatchSuspendtRSRatio)
|
||||
};
|
||||
|
||||
// Action to log more info for debugging
|
||||
|
|
@ -58,4 +68,16 @@ public class ServerKillingMonkeyFactory extends MonkeyFactory {
|
|||
new PeriodicRandomActionPolicy(60 * 1000, actions1)),
|
||||
new PeriodicRandomActionPolicy(60 * 1000, actions2));
|
||||
}
|
||||
|
||||
private void loadProperties() {
|
||||
gracefulRollingRestartTSSLeepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.GRACEFUL_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_GRACEFUL_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchSuspendRSSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchSuspendtRSRatio = Float.parseFloat(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_RATIO,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO + ""));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.chaos.actions.DecreaseMaxHFileSizeAction;
|
|||
import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.FlushTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.GracefulRollingRestartRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction;
|
||||
|
|
@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.chaos.actions.RestartActiveMasterAction;
|
|||
import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RollingBatchSuspendResumeRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.SnapshotTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.SplitAllRegionOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction;
|
||||
|
|
@ -66,6 +68,9 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory {
|
|||
private float compactTableRatio;
|
||||
private float compactRandomRegionRatio;
|
||||
private long decreaseHFileSizeSleepTime;
|
||||
private long gracefulRollingRestartTSSLeepTime;
|
||||
private long rollingBatchSuspendRSSleepTime;
|
||||
private float rollingBatchSuspendtRSRatio;
|
||||
|
||||
@Override
|
||||
public ChaosMonkey build() {
|
||||
|
|
@ -110,6 +115,9 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory {
|
|||
new RestartRsHoldingMetaAction(restartRsHoldingMetaSleepTime),
|
||||
new DecreaseMaxHFileSizeAction(decreaseHFileSizeSleepTime, tableName),
|
||||
new SplitAllRegionOfTableAction(tableName),
|
||||
new GracefulRollingRestartRsAction(gracefulRollingRestartTSSLeepTime),
|
||||
new RollingBatchSuspendResumeRsAction(rollingBatchSuspendRSSleepTime,
|
||||
rollingBatchSuspendtRSRatio)
|
||||
};
|
||||
|
||||
// Action to log more info for debugging
|
||||
|
|
@ -179,5 +187,14 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory {
|
|||
decreaseHFileSizeSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.DECREASE_HFILE_SIZE_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME + ""));
|
||||
gracefulRollingRestartTSSLeepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.GRACEFUL_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_GRACEFUL_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchSuspendRSSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchSuspendtRSRatio = Float.parseFloat(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_RATIO,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO + ""));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.chaos.actions.DecreaseMaxHFileSizeAction;
|
|||
import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.FlushTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.GracefulRollingRestartRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction;
|
||||
|
|
@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.chaos.actions.RemoveColumnAction;
|
|||
import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingMetaAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RollingBatchRestartRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.RollingBatchSuspendResumeRsAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.SplitAllRegionOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction;
|
||||
import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey;
|
||||
|
|
@ -43,8 +45,15 @@ import org.apache.hadoop.hbase.chaos.policies.DoActionsOncePolicy;
|
|||
import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy;
|
||||
|
||||
public class StressAssignmentManagerMonkeyFactory extends MonkeyFactory {
|
||||
|
||||
private long gracefulRollingRestartTSSLeepTime;
|
||||
private long rollingBatchSuspendRSSleepTime;
|
||||
private float rollingBatchSuspendtRSRatio;
|
||||
|
||||
@Override
|
||||
public ChaosMonkey build() {
|
||||
loadProperties();
|
||||
|
||||
// Actions that could slow down region movement.
|
||||
// These could also get regions stuck if there are issues.
|
||||
Action[] actions1 = new Action[]{
|
||||
|
|
@ -72,6 +81,9 @@ public class StressAssignmentManagerMonkeyFactory extends MonkeyFactory {
|
|||
new SplitAllRegionOfTableAction(tableName),
|
||||
new DecreaseMaxHFileSizeAction(MonkeyConstants.DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME,
|
||||
tableName),
|
||||
new GracefulRollingRestartRsAction(gracefulRollingRestartTSSLeepTime),
|
||||
new RollingBatchSuspendResumeRsAction(rollingBatchSuspendRSSleepTime,
|
||||
rollingBatchSuspendtRSRatio)
|
||||
};
|
||||
|
||||
// Action to log more info for debugging
|
||||
|
|
@ -87,4 +99,16 @@ public class StressAssignmentManagerMonkeyFactory extends MonkeyFactory {
|
|||
new PeriodicRandomActionPolicy(90 * 1000, actions3)
|
||||
);
|
||||
}
|
||||
|
||||
private void loadProperties() {
|
||||
gracefulRollingRestartTSSLeepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.GRACEFUL_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_GRACEFUL_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchSuspendRSSleepTime = Long.parseLong(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_SLEEP_TIME,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_SLEEP_TIME + ""));
|
||||
rollingBatchSuspendtRSRatio = Float.parseFloat(this.properties.getProperty(
|
||||
MonkeyConstants.ROLLING_BATCH_RESTART_RS_RATIO,
|
||||
MonkeyConstants.DEFAULT_ROLLING_BATCH_RESTART_RS_RATIO + ""));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,9 @@
|
|||
|
||||
package org.apache.hadoop.hbase.chaos.monkies;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.RandomUtils;
|
||||
|
|
@ -90,18 +91,13 @@ public class PolicyBasedChaosMonkey extends ChaosMonkey {
|
|||
|
||||
/** Selects and returns ceil(ratio * items.length) random items from the given array */
|
||||
public static <T> List<T> selectRandomItems(T[] items, float ratio) {
|
||||
int remaining = (int)Math.ceil(items.length * ratio);
|
||||
int selectedNumber = (int)Math.ceil(items.length * ratio);
|
||||
|
||||
List<T> selectedItems = new ArrayList<>(remaining);
|
||||
List<T> originalItems = Arrays.asList(items);
|
||||
Collections.shuffle(originalItems);
|
||||
|
||||
for (int i=0; i<items.length && remaining > 0; i++) {
|
||||
if (RandomUtils.nextFloat() < ((float)remaining/(items.length-i))) {
|
||||
selectedItems.add(items[i]);
|
||||
remaining--;
|
||||
}
|
||||
}
|
||||
|
||||
return selectedItems;
|
||||
int startIndex = RandomUtils.nextInt(0, items.length - selectedNumber);
|
||||
return originalItems.subList(startIndex, startIndex + selectedNumber);
|
||||
}
|
||||
|
||||
private Policy[] policies;
|
||||
|
|
|
|||
|
|
@ -148,6 +148,20 @@ public abstract class HBaseCluster implements Closeable, Configurable {
|
|||
public abstract void waitForRegionServerToStop(ServerName serverName, long timeout)
|
||||
throws IOException;
|
||||
|
||||
/**
|
||||
* Suspend the region server
|
||||
* @param serverName the hostname to suspend the regionserver on
|
||||
* @throws IOException if something goes wrong
|
||||
*/
|
||||
public abstract void suspendRegionServer(ServerName serverName) throws IOException;
|
||||
|
||||
/**
|
||||
* Resume the region server
|
||||
* @param serverName the hostname to resume the regionserver on
|
||||
* @throws IOException if something goes wrong
|
||||
*/
|
||||
public abstract void resumeRegionServer(ServerName serverName) throws IOException;
|
||||
|
||||
/**
|
||||
* Starts a new zookeeper node on the given hostname or if this is a mini/local cluster,
|
||||
* silently logs warning message.
|
||||
|
|
|
|||
|
|
@ -291,6 +291,16 @@ public class MiniHBaseCluster extends HBaseCluster {
|
|||
stopRegionServer(getRegionServerIndex(serverName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void suspendRegionServer(ServerName serverName) throws IOException {
|
||||
suspendRegionServer(getRegionServerIndex(serverName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void resumeRegionServer(ServerName serverName) throws IOException {
|
||||
resumeRegionServer(getRegionServerIndex(serverName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void waitForRegionServerToStop(ServerName serverName, long timeout) throws IOException {
|
||||
//ignore timeout for now
|
||||
|
|
@ -489,6 +499,32 @@ public class MiniHBaseCluster extends HBaseCluster {
|
|||
return server;
|
||||
}
|
||||
|
||||
/**
|
||||
* Suspend the specified region server
|
||||
* @param serverNumber Used as index into a list.
|
||||
* @return
|
||||
*/
|
||||
public JVMClusterUtil.RegionServerThread suspendRegionServer(int serverNumber) {
|
||||
JVMClusterUtil.RegionServerThread server =
|
||||
hbaseCluster.getRegionServers().get(serverNumber);
|
||||
LOG.info("Suspending {}", server.toString());
|
||||
server.suspend();
|
||||
return server;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume the specified region server
|
||||
* @param serverNumber Used as index into a list.
|
||||
* @return
|
||||
*/
|
||||
public JVMClusterUtil.RegionServerThread resumeRegionServer(int serverNumber) {
|
||||
JVMClusterUtil.RegionServerThread server =
|
||||
hbaseCluster.getRegionServers().get(serverNumber);
|
||||
LOG.info("Resuming {}", server.toString());
|
||||
server.resume();
|
||||
return server;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the specified region server to stop. Removes this thread from list
|
||||
* of running threads.
|
||||
|
|
|
|||
Loading…
Reference in New Issue