HBASE-16008 A robust way deal with early termination of HBCK (Stephen Yuan Jiang)

This commit is contained in:
Stephen Yuan Jiang 2016-07-23 14:47:11 -07:00
parent cd0b85e0a8
commit bdd7782f05
15 changed files with 1738 additions and 652 deletions

View File

@ -958,6 +958,13 @@ public interface Admin extends Abortable, Closeable {
*/
void stopMaster() throws IOException;
/**
* Check whether Master is in maintenance mode
*
* @throws IOException if a remote or network exception occurs
*/
boolean isMasterInMaintenanceMode() throws IOException;
/**
* Stop the designated regionserver
*

View File

@ -1415,6 +1415,13 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
return stub.stopMaster(controller, request);
}
@Override
public MasterProtos.IsInMaintenanceModeResponse isMasterInMaintenanceMode(
final RpcController controller,
final MasterProtos.IsInMaintenanceModeRequest request) throws ServiceException {
return stub.isMasterInMaintenanceMode(controller, request);
}
@Override
public MasterProtos.BalanceResponse balance(RpcController controller,
MasterProtos.BalanceRequest request) throws ServiceException {

View File

@ -130,6 +130,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterSta
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsInMaintenanceModeRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsInMaintenanceModeResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
@ -1932,6 +1934,19 @@ public class HBaseAdmin implements Admin {
}
}
@Override
public boolean isMasterInMaintenanceMode() throws IOException {
return executeCallable(new MasterCallable<IsInMaintenanceModeResponse>(getConnection()) {
@Override
public IsInMaintenanceModeResponse call(int callTimeout) throws ServiceException {
PayloadCarryingRpcController controller = rpcControllerFactory.newController();
controller.setCallTimeout(callTimeout);
return master.isMasterInMaintenanceMode(
controller, IsInMaintenanceModeRequest.newBuilder().build());
}
}).getInMaintenanceMode();
}
@Override
public ClusterStatus getClusterStatus() throws IOException {
return executeCallable(new MasterCallable<ClusterStatus>(getConnection()) {

View File

@ -123,6 +123,8 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
public String recoveringRegionsZNode;
// znode containing namespace descriptors
public static String namespaceZNode = "namespace";
// znode of indicating master maintenance mode
public static String masterMaintZNode = "masterMaintenance";
public final static String META_ZNODE_PREFIX = "meta-region-server";
@ -194,6 +196,7 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
ZKUtil.createAndFailSilent(this, backupMasterAddressesZNode);
ZKUtil.createAndFailSilent(this, tableLockZNode);
ZKUtil.createAndFailSilent(this, recoveringRegionsZNode);
ZKUtil.createAndFailSilent(this, masterMaintZNode);
} catch (KeeperException e) {
throw new ZooKeeperConnectionException(
prefix("Unexpected KeeperException creating base node"), e);
@ -442,6 +445,8 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
conf.get("zookeeper.znode.recovering.regions", "recovering-regions"));
namespaceZNode = ZKUtil.joinZNode(baseZNode,
conf.get("zookeeper.znode.namespace", "namespace"));
masterMaintZNode = ZKUtil.joinZNode(baseZNode,
conf.get("zookeeper.znode.masterMaintenance", "master-maintenance"));
}
/**

View File

@ -258,6 +258,13 @@ message StopMasterRequest {
message StopMasterResponse {
}
message IsInMaintenanceModeRequest {
}
message IsInMaintenanceModeResponse {
required bool inMaintenanceMode = 1;
}
message BalanceRequest {
optional bool force = 1;
}
@ -640,6 +647,12 @@ service MasterService {
rpc StopMaster(StopMasterRequest)
returns(StopMasterResponse);
/**
* Query whether the Master is in maintenance mode.
*/
rpc IsMasterInMaintenanceMode(IsInMaintenanceModeRequest)
returns(IsInMaintenanceModeResponse);
/**
* Run the balancer. Will run the balancer and if regions to move, it will
* go ahead and do the reassignments. Can NOT run for various reasons.

View File

@ -109,6 +109,7 @@ public class CatalogJanitor extends ScheduledChore {
try {
AssignmentManager am = this.services.getAssignmentManager();
if (this.enabled.get()
&& !this.services.isInMaintenanceMode()
&& am != null
&& am.isFailoverCleanupDone()
&& am.getRegionStates().getRegionsInTransition().size() == 0) {
@ -241,6 +242,11 @@ public class CatalogJanitor extends ScheduledChore {
int mergeCleaned = 0;
Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond();
for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) {
if (this.services.isInMaintenanceMode()) {
// Stop cleaning if the master is in maintenance mode
break;
}
PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(e.getValue());
HRegionInfo regionA = p.getFirst();
HRegionInfo regionB = p.getSecond();
@ -266,6 +272,11 @@ public class CatalogJanitor extends ScheduledChore {
// regions whose parents are still around
HashSet<String> parentNotCleaned = new HashSet<String>();
for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
if (this.services.isInMaintenanceMode()) {
// Stop cleaning if the master is in maintenance mode
break;
}
if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
cleanParent(e.getKey(), e.getValue())) {
splitCleaned++;

View File

@ -159,6 +159,8 @@ import org.apache.hadoop.hbase.util.ZKDataMigrator;
import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
import org.apache.hadoop.hbase.zookeeper.MasterMaintenanceModeTracker;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker;
import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker;
@ -261,6 +263,9 @@ public class HMaster extends HRegionServer implements MasterServices {
// Tracker for region normalizer state
private RegionNormalizerTracker regionNormalizerTracker;
//Tracker for master maintenance mode setting
private MasterMaintenanceModeTracker maintenanceModeTracker;
private ClusterSchemaService clusterSchemaService;
// Metrics for the HMaster
@ -616,6 +621,9 @@ public class HMaster extends HRegionServer implements MasterServices {
this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this, this.serverManager);
this.drainingServerTracker.start();
this.maintenanceModeTracker = new MasterMaintenanceModeTracker(zooKeeper);
this.maintenanceModeTracker.start();
// Set the cluster as up. If new RSs, they'll be waiting on this before
// going ahead with their startup.
boolean wasUp = this.clusterStatusTracker.isClusterUp();
@ -1125,6 +1133,12 @@ public class HMaster extends HRegionServer implements MasterServices {
LOG.debug("Master has not been initialized, don't run balancer.");
return false;
}
if (isInMaintenanceMode()) {
LOG.info("Master is in maintenanceMode mode, don't run balancer.");
return false;
}
// Do this call outside of synchronized block.
int maximumBalanceTime = getBalancerCutoffTime();
synchronized (this.balancer) {
@ -1228,6 +1242,11 @@ public class HMaster extends HRegionServer implements MasterServices {
return false;
}
if (isInMaintenanceMode()) {
LOG.info("Master is in maintenance mode, don't run region normalizer.");
return false;
}
if (!this.regionNormalizerTracker.isNormalizerOn()) {
LOG.debug("Region normalization is disabled, don't run region normalizer.");
return false;
@ -1241,6 +1260,11 @@ public class HMaster extends HRegionServer implements MasterServices {
Collections.shuffle(allEnabledTables);
for (TableName table : allEnabledTables) {
if (isInMaintenanceMode()) {
LOG.debug("Master is in maintenance mode, stop running region normalizer.");
return false;
}
HTableDescriptor tblDesc = getTableDescriptors().get(table);
if (table.isSystemTable() || (tblDesc != null &&
!tblDesc.isNormalizationEnabled())) {
@ -2264,6 +2288,16 @@ public class HMaster extends HRegionServer implements MasterServices {
return initialized.isReady();
}
/**
* Report whether this master is in maintenance mode.
*
* @return true if master is in maintenanceMode
*/
@Override
public boolean isInMaintenanceMode() {
return maintenanceModeTracker.isInMaintenanceMode();
}
@VisibleForTesting
public void setInitialized(boolean isInitialized) {
procedureExecutor.getEnvironment().setEventReady(initialized, isInitialized);
@ -2735,7 +2769,9 @@ public class HMaster extends HRegionServer implements MasterServices {
* @return The state of the load balancer, or false if the load balancer isn't defined.
*/
public boolean isBalancerOn() {
if (null == loadBalancerTracker) return false;
if (null == loadBalancerTracker || isInMaintenanceMode()) {
return false;
}
return loadBalancerTracker.isBalancerOn();
}
@ -2744,10 +2780,10 @@ public class HMaster extends HRegionServer implements MasterServices {
* false is returned.
*/
public boolean isNormalizerOn() {
return null == regionNormalizerTracker? false: regionNormalizerTracker.isNormalizerOn();
return (null == regionNormalizerTracker || isInMaintenanceMode()) ?
false: regionNormalizerTracker.isNormalizerOn();
}
/**
* Queries the state of the {@link SplitOrMergeTracker}. If it is not initialized,
* false is returned. If switchType is illegal, false will return.
@ -2755,7 +2791,7 @@ public class HMaster extends HRegionServer implements MasterServices {
* @return The state of the switch
*/
public boolean isSplitOrMergeEnabled(MasterSwitchType switchType) {
if (null == splitOrMergeTracker) {
if (null == splitOrMergeTracker || isInMaintenanceMode()) {
return false;
}
return splitOrMergeTracker.isSplitOrMergeEnabled(switchType);

View File

@ -1245,6 +1245,15 @@ public class MasterRpcServices extends RSRpcServices
return StopMasterResponse.newBuilder().build();
}
@Override
public IsInMaintenanceModeResponse isMasterInMaintenanceMode(
final RpcController controller,
final IsInMaintenanceModeRequest request) throws ServiceException {
IsInMaintenanceModeResponse.Builder response = IsInMaintenanceModeResponse.newBuilder();
response.setInMaintenanceMode(master.isInMaintenanceMode());
return response.build();
}
@Override
public UnassignRegionResponse unassignRegion(RpcController controller,
UnassignRegionRequest req) throws ServiceException {

View File

@ -309,6 +309,11 @@ public interface MasterServices extends Server {
*/
boolean isInitialized();
/**
* @return true if master is in maintanceMode
*/
boolean isInMaintenanceMode();
/**
* Abort a procedure.
* @param procId ID of the procedure

View File

@ -104,7 +104,6 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
@ -209,6 +208,9 @@ public class HBaseFsck extends Configured implements Closeable {
// AlreadyBeingCreatedException which is implies timeout on this operations up to
// HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds).
private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds
private static final int DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS = 5;
private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds
private static final int DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds
/**********************
* Internal resources
@ -236,8 +238,6 @@ public class HBaseFsck extends Configured implements Closeable {
private static boolean details = false; // do we display the full report
private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
private static boolean forceExclusive = false; // only this hbck can modify HBase
private static boolean disableBalancer = false; // disable load balancer to keep regions stable
private static boolean disableSplitAndMerge = false; // disable split and merge
private boolean fixAssignments = false; // fix assignment errors?
private boolean fixMeta = false; // fix meta errors?
private boolean checkHdfs = true; // load and check fs consistency?
@ -306,10 +306,13 @@ public class HBaseFsck extends Configured implements Closeable {
private Map<TableName, TableState> tableStates =
new HashMap<TableName, TableState>();
private final RetryCounterFactory lockFileRetryCounterFactory;
private final RetryCounterFactory createZNodeRetryCounterFactory;
private Map<TableName, Set<String>> skippedRegions = new HashMap<TableName, Set<String>>();
ZooKeeperWatcher zkw = null;
private ZooKeeperWatcher zkw = null;
private String hbckEphemeralNodePath = null;
private boolean hbckZodeCreated = false;
/**
* Constructor
@ -349,6 +352,14 @@ public class HBaseFsck extends Configured implements Closeable {
"hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL),
getConf().getInt(
"hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME));
createZNodeRetryCounterFactory = new RetryCounterFactory(
getConf().getInt("hbase.hbck.createznode.attempts", DEFAULT_MAX_CREATE_ZNODE_ATTEMPTS),
getConf().getInt(
"hbase.hbck.createznode.attempt.sleep.interval",
DEFAULT_CREATE_ZNODE_ATTEMPT_SLEEP_INTERVAL),
getConf().getInt(
"hbase.hbck.createznode.attempt.maxsleeptime",
DEFAULT_CREATE_ZNODE_ATTEMPT_MAX_SLEEP_TIME));
zkw = createZooKeeperWatcher();
}
@ -498,6 +509,7 @@ public class HBaseFsck extends Configured implements Closeable {
@Override
public void run() {
IOUtils.closeQuietly(HBaseFsck.this);
cleanupHbckZnode();
unlockHbck();
}
});
@ -676,49 +688,78 @@ public class HBaseFsck extends Configured implements Closeable {
return errors.getErrorList().size();
}
/**
* This method maintains an ephemeral znode. If the creation fails we return false or throw
* exception
*
* @return true if creating znode succeeds; false otherwise
* @throws IOException if IO failure occurs
*/
private boolean setMasterInMaintenanceMode() throws IOException {
RetryCounter retryCounter = createZNodeRetryCounterFactory.create();
hbckEphemeralNodePath = ZKUtil.joinZNode(
ZooKeeperWatcher.masterMaintZNode,
"hbck-" + Long.toString(EnvironmentEdgeManager.currentTime()));
do {
try {
hbckZodeCreated = ZKUtil.createEphemeralNodeAndWatch(zkw, hbckEphemeralNodePath, null);
if (hbckZodeCreated) {
break;
}
} catch (KeeperException e) {
if (retryCounter.getAttemptTimes() >= retryCounter.getMaxAttempts()) {
throw new IOException("Can't create znode " + hbckEphemeralNodePath, e);
}
// fall through and retry
}
LOG.warn("Fail to create znode " + hbckEphemeralNodePath + ", try=" +
(retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts());
try {
retryCounter.sleepUntilNextRetry();
} catch (InterruptedException ie) {
throw (InterruptedIOException) new InterruptedIOException(
"Can't create znode " + hbckEphemeralNodePath).initCause(ie);
}
} while (retryCounter.shouldRetry());
return hbckZodeCreated;
}
private void cleanupHbckZnode() {
try {
if (zkw != null && hbckZodeCreated) {
ZKUtil.deleteNode(zkw, hbckEphemeralNodePath);
hbckZodeCreated = false;
}
} catch (KeeperException e) {
// Ignore
if (!e.code().equals(KeeperException.Code.NONODE)) {
LOG.warn("Delete HBCK znode " + hbckEphemeralNodePath + " failed ", e);
}
}
}
/**
* Contacts the master and prints out cluster-wide information
* @return 0 on success, non-zero on failure
*/
public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
public int onlineHbck()
throws IOException, KeeperException, InterruptedException, ServiceException {
// print hbase server version
errors.print("Version: " + status.getHBaseVersion());
offlineHdfsIntegrityRepair();
boolean oldBalancer = false;
if (shouldDisableBalancer()) {
oldBalancer = admin.setBalancerRunning(false, true);
}
boolean[] oldSplitAndMerge = null;
if (shouldDisableSplitAndMerge()) {
oldSplitAndMerge = admin.setSplitOrMergeEnabled(false, false,
MasterSwitchType.SPLIT, MasterSwitchType.MERGE);
// If Master runs maintenance tasks (such as balancer, catalog janitor, etc) during online
// hbck, it is likely that hbck would be misled and report transient errors. Therefore, it
// is better to set Master into maintenance mode during online hbck.
//
if (!setMasterInMaintenanceMode()) {
LOG.warn("HBCK is running while master is not in maintenance mode, you might see transient "
+ "error. Please run HBCK multiple times to reduce the chance of transient error.");
}
try {
onlineConsistencyRepair();
}
finally {
// Only restore the balancer if it was true when we started repairing and
// we actually disabled it. Otherwise, we might clobber another run of
// hbck that has just restored it.
if (shouldDisableBalancer() && oldBalancer) {
admin.setBalancerRunning(oldBalancer, false);
}
if (shouldDisableSplitAndMerge()) {
if (oldSplitAndMerge != null) {
if (oldSplitAndMerge[0] && oldSplitAndMerge[1]) {
admin.setSplitOrMergeEnabled(true, false,
MasterSwitchType.SPLIT, MasterSwitchType.MERGE);
} else if (oldSplitAndMerge[0]) {
admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.SPLIT);
} else if (oldSplitAndMerge[1]) {
admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.MERGE);
}
}
}
}
if (checkRegionBoundaries) {
checkRegionBoundaries();
@ -730,6 +771,9 @@ public class HBaseFsck extends Configured implements Closeable {
checkAndFixReplication();
// Remove the hbck znode
cleanupHbckZnode();
// Remove the hbck lock
unlockHbck();
@ -750,6 +794,7 @@ public class HBaseFsck extends Configured implements Closeable {
@Override
public void close() throws IOException {
try {
cleanupHbckZnode();
unlockHbck();
} catch (Exception io) {
LOG.warn(io);
@ -4221,38 +4266,6 @@ public class HBaseFsck extends Configured implements Closeable {
return fixAny || forceExclusive;
}
/**
* Disable the load balancer.
*/
public static void setDisableBalancer() {
disableBalancer = true;
}
/**
* Disable the split and merge
*/
public static void setDisableSplitAndMerge() {
disableSplitAndMerge = true;
}
/**
* The balancer should be disabled if we are modifying HBase.
* It can be disabled if you want to prevent region movement from causing
* false positives.
*/
public boolean shouldDisableBalancer() {
return fixAny || disableBalancer;
}
/**
* The split and merge should be disabled if we are modifying HBase.
* It can be disabled if you want to prevent region movement from causing
* false positives.
*/
public boolean shouldDisableSplitAndMerge() {
return fixAny || disableSplitAndMerge;
}
/**
* Set summary mode.
* Print only summary of the tables and status (OK or INCONSISTENT)
@ -4514,7 +4527,6 @@ public class HBaseFsck extends Configured implements Closeable {
out.println(" -sidelineDir <hdfs://> HDFS path to backup existing meta.");
out.println(" -boundaries Verify that regions boundaries are the same between META and store files.");
out.println(" -exclusive Abort if another hbck is exclusive or fixing.");
out.println(" -disableBalancer Disable the load balancer.");
out.println("");
out.println(" Metadata Repair options: (expert features, use with caution!)");
@ -4610,10 +4622,6 @@ public class HBaseFsck extends Configured implements Closeable {
setDisplayFullReport();
} else if (cmd.equals("-exclusive")) {
setForceExclusive();
} else if (cmd.equals("-disableBalancer")) {
setDisableBalancer();
} else if (cmd.equals("-disableSplitAndMerge")) {
setDisableSplitAndMerge();
} else if (cmd.equals("-timelag")) {
if (i == args.length - 1) {
errors.reportError(ERROR_CODE.WRONG_USAGE, "HBaseFsck: -timelag needs a value.");

View File

@ -0,0 +1,81 @@
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.zookeeper;
import java.util.List;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
/**
* Tracks the master Maintenance Mode via ZK.
*/
@InterfaceAudience.Private
public class MasterMaintenanceModeTracker extends ZooKeeperListener {
private boolean hasChildren;
public MasterMaintenanceModeTracker(ZooKeeperWatcher watcher) {
super(watcher);
hasChildren = false;
}
public boolean isInMaintenanceMode() {
return hasChildren;
}
private void update(String path) {
if (path.startsWith(ZooKeeperWatcher.masterMaintZNode)) {
update();
}
}
private void update() {
try {
List<String> children =
ZKUtil.listChildrenAndWatchForNewChildren(watcher, ZooKeeperWatcher.masterMaintZNode);
hasChildren = (children != null && children.size() > 0);
} catch (KeeperException e) {
// Ignore the ZK keeper exception
hasChildren = false;
}
}
/**
* Starts the tracking of whether master is in Maintenance Mode.
*/
public void start() {
watcher.registerListener(this);
update();
}
@Override
public void nodeCreated(String path) {
update(path);
}
@Override
public void nodeDeleted(String path) {
update(path);
}
@Override
public void nodeChildrenChanged(String path) {
update(path);
}
}

View File

@ -146,6 +146,4 @@ public class SplitOrMergeTracker {
return builder.build();
}
}
}

View File

@ -306,6 +306,11 @@ public class MockNoopMasterServices implements MasterServices, Server {
return false;
}
@Override
public boolean isInMaintenanceMode() {
return false;
}
@Override
public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
return 0;

View File

@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
@ -78,7 +77,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;