diff --git a/CHANGES.txt b/CHANGES.txt index 199707c42d7..eda3c92694c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -384,6 +384,8 @@ Release 0.92.0 - Unreleased HBASE-4510 Check and workaround usage of internal HDFS APIs in HBase (Harsh) HBASE-4595 HFilePrettyPrinter Scanned kv count always 0 (Matteo Bertozzi) + HBASE-4580 Some invalid zk nodes were created when a clean cluster restarts + (Gaojinchao) TESTS HBASE-4450 test for number of blocks read: to serve as baseline for expected diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index cda50919596..def25db32a3 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -314,11 +314,8 @@ public class AssignmentManager extends ZooKeeperListener { // Returns servers who have not checked in (assumed dead) and their regions Map>> deadServers = rebuildUserRegions(); - // Process list of dead servers; note this will add regions to the RIT. - // processRegionsInTransition will read them and assign them out. - processDeadServers(deadServers); - // Check existing regions in transition - processRegionsInTransition(deadServers); + + processDeadServersAndRegionsInTransition(deadServers); // Recover the tables that were not fully moved to DISABLED state. // These tables are in DISABLING state when the master restarted/switched. @@ -333,21 +330,23 @@ public class AssignmentManager extends ZooKeeperListener { * @throws IOException * @throws InterruptedException */ - void processRegionsInTransition() + void processDeadServersAndRegionsInTransition() throws KeeperException, IOException, InterruptedException { // Pass null to signify no dead servers in this context. - processRegionsInTransition(null); + processDeadServersAndRegionsInTransition(null); } /** - * Process all regions that are in transition up in zookeeper. Used by - * master joining an already running cluster. - * @param deadServers Map of dead servers and their regions. Can be null. + * Process all regions that are in transition in zookeeper and also + * processes the list of dead servers by scanning the META. + * Used by master joining an cluster. + * @param deadServers + * Map of dead servers and their regions. Can be null. * @throws KeeperException * @throws IOException * @throws InterruptedException */ - void processRegionsInTransition( + void processDeadServersAndRegionsInTransition( final Map>> deadServers) throws KeeperException, IOException, InterruptedException { List nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher, @@ -374,11 +373,10 @@ public class AssignmentManager extends ZooKeeperListener { // If we found user regions out on cluster, its a failover. if (regionsToProcess) { LOG.info("Found regions out on cluster or in RIT; failover"); - if (!nodes.isEmpty()) { - for (String encodedRegionName: nodes) { - processRegionInTransition(encodedRegionName, null, deadServers); - } - } + // Process list of dead servers and regions in RIT. + // See HBASE-4580 for more information. + processDeadServersAndRecoverLostRegions(deadServers, nodes); + } else { // Fresh cluster startup. LOG.info("Clean cluster startup. Assigning userregions"); @@ -2222,54 +2220,69 @@ public class AssignmentManager extends ZooKeeperListener { } /** - * Processes list of dead servers from result of META scan. + * Processes list of dead servers from result of META scan and regions in RIT *

- * This is used as part of failover to handle RegionServers which failed - * while there was no active master. + * This is used for failover to recover the lost regions that belonged to + * RegionServers which failed while there was no active master or regions + * that were in RIT. *

- * Method stubs in-memory data to be as expected by the normal server shutdown - * handler. - * + * * @param deadServers + * The list of dead servers which failed while there was no active + * master. Can be null. + * @param nodes + * The regions in RIT * @throws IOException * @throws KeeperException */ - private void processDeadServers( - Map>> deadServers) - throws IOException, KeeperException { - for (Map.Entry>> deadServer: + private void processDeadServersAndRecoverLostRegions( + Map>> deadServers, + List nodes) throws IOException, KeeperException { + if (null != deadServers) { + for (Map.Entry>> deadServer : deadServers.entrySet()) { - List> regions = deadServer.getValue(); - for (Pair region : regions) { - HRegionInfo regionInfo = region.getFirst(); - Result result = region.getSecond(); - // If region was in transition (was in zk) force it offline for reassign - try { - RegionTransitionData data = ZKAssign.getData(watcher, - regionInfo.getEncodedName()); + List> regions = deadServer.getValue(); + for (Pair region : regions) { + HRegionInfo regionInfo = region.getFirst(); + Result result = region.getSecond(); + // If region was in transition (was in zk) force it offline for + // reassign + try { + RegionTransitionData data = ZKAssign.getData(watcher, + regionInfo.getEncodedName()); - // If zk node of this region has been updated by a live server, - // we consider that this region is being handled. - // So we should skip it and process it in processRegionsInTransition. - if (data != null && data.getOrigin() != null && - serverManager.isServerOnline(data.getOrigin())) { - LOG.info("The region " + regionInfo.getEncodedName() - + "is being handled on " + data.getOrigin()); - continue; + // If zk node of this region has been updated by a live server, + // we consider that this region is being handled. + // So we should skip it and process it in + // processRegionsInTransition. + if (data != null && data.getOrigin() != null && + serverManager.isServerOnline(data.getOrigin())) { + LOG.info("The region " + regionInfo.getEncodedName() + + "is being handled on " + data.getOrigin()); + continue; + } + // Process with existing RS shutdown code + boolean assign = ServerShutdownHandler.processDeadRegion( + regionInfo, result, this, this.catalogTracker); + if (assign) { + ZKAssign.createOrForceNodeOffline(watcher, regionInfo, + master.getServerName()); + if (!nodes.contains(regionInfo.getEncodedName())) { + nodes.add(regionInfo.getEncodedName()); + } + } + } catch (KeeperException.NoNodeException nne) { + // This is fine } - // Process with existing RS shutdown code - boolean assign = - ServerShutdownHandler.processDeadRegion(regionInfo, result, this, - this.catalogTracker); - if (assign) { - ZKAssign.createOrForceNodeOffline(watcher, regionInfo, - master.getServerName()); - } - } catch (KeeperException.NoNodeException nne) { - // This is fine } } } + + if (!nodes.isEmpty()) { + for (String encodedRegionName : nodes) { + processRegionInTransition(encodedRegionName, null, deadServers); + } + } } /* diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 50b49a6e6e7..f5bd130a1b4 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1201,7 +1201,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { // process RIT if any // TODO: Why does this not call AssignmentManager.joinCluster? Otherwise // we are not processing dead servers if any. - this.assignmentManager.processRegionsInTransition(); + this.assignmentManager.processDeadServersAndRegionsInTransition(); return true; } finally { status.cleanup();