From 3523b66eece09f9a68c3b340c09ad831b413e1fb Mon Sep 17 00:00:00 2001 From: Jonathan Gray Date: Thu, 28 Oct 2010 21:31:58 +0000 Subject: [PATCH] HBASE-3159 Double play of OpenedRegionHandler for a single region and assorted fixes around this + TestRollingRestart added git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1028497 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 3 + .../hadoop/hbase/LocalHBaseCluster.java | 52 +++ .../hadoop/hbase/catalog/CatalogTracker.java | 2 + .../hbase/master/AssignmentManager.java | 17 +- .../apache/hadoop/hbase/master/HMaster.java | 6 +- .../hadoop/hbase/master/ServerManager.java | 2 +- .../master/handler/ClosedRegionHandler.java | 4 +- .../master/handler/OpenedRegionHandler.java | 3 +- .../hbase/regionserver/HRegionServer.java | 2 + .../hadoop/hbase/zookeeper/ZKAssign.java | 41 +- .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 5 +- .../hadoop/hbase/HBaseTestingUtility.java | 24 ++ .../hbase/master/TestRollingRestart.java | 353 ++++++++++++++++++ 13 files changed, 495 insertions(+), 19 deletions(-) create mode 100644 src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java diff --git a/CHANGES.txt b/CHANGES.txt index 5a3fb778ada..cf58b9047fd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -620,6 +620,9 @@ Release 0.21.0 - Unreleased HBASE-3155 HFile.appendMetaBlock() uses wrong comparator (Nicolas Spiegelberg via Stack) HBASE-3012 TOF doesn't take zk client port for remote clusters + HBASE-3159 Double play of OpenedRegionHandler for a single region + and assorted fixes around this + TestRollingRestart added + IMPROVEMENTS HBASE-1760 Cleanup TODOs in HTable diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java index bd0944ee0de..926767951a9 100644 --- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java +++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java @@ -238,6 +238,33 @@ public class LocalHBaseCluster { return regionServerThread.getName(); } + /** + * Wait for the specified region server to stop + * Removes this thread from list of running threads. + * @param serverNumber + * @return Name of region server that just went down. + */ + public String waitOnRegionServer(JVMClusterUtil.RegionServerThread rst) { + while (rst.isAlive()) { + try { + LOG.info("Waiting on " + + rst.getRegionServer().getHServerInfo().toString()); + rst.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + for (int i=0;i + * Used in testing only. * @param zkw zk reference * @throws KeeperException * @throws InterruptedException @@ -759,7 +774,27 @@ public class ZKAssign { LOG.debug("ZK RIT -> " + znode); } } - Thread.sleep(200); + Thread.sleep(100); + } + } + + /** + * Blocks until there is at least one node in regions in transition. + *

+ * Used in testing only. + * @param zkw zk reference + * @throws KeeperException + * @throws InterruptedException + */ + public static void blockUntilRIT(ZooKeeperWatcher zkw) + throws KeeperException, InterruptedException { + while (!ZKUtil.nodeHasChildren(zkw, zkw.assignmentZNode)) { + List znodes = + ZKUtil.listChildrenAndWatchForNewChildren(zkw, zkw.assignmentZNode); + if (znodes == null || znodes.isEmpty()) { + LOG.debug("No RIT in ZK"); + } + Thread.sleep(100); } } diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index 565d9d3f574..7cd1c3944f7 100644 --- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -1093,6 +1093,9 @@ public class ZKUtil { LOG.debug(zkw.prefix("Retrieved " + ((data == null)? 0: data.length) + " byte(s) of data from znode " + znode + (watcherSet? " and set watcher; ": "; data=") + - (data == null? "null": StringUtils.abbreviate(Bytes.toString(data), 32)))); + (data == null? "null": ( + znode.startsWith(zkw.assignmentZNode) ? + RegionTransitionData.fromBytes(data).toString() + : StringUtils.abbreviate(Bytes.toString(data), 32))))); } } diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 2c1cba581f2..83ceac97d7e 100644 --- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -715,6 +715,30 @@ public class HBaseTestingUtility { return createMultiRegions(c, table, columnFamily, KEYS); } + /** + * Creates the specified number of regions in the specified table. + * @param c + * @param table + * @param columnFamily + * @param startKeys + * @return + * @throws IOException + */ + public int createMultiRegions(final Configuration c, final HTable table, + final byte [] family, int numRegions) + throws IOException { + if (numRegions < 3) throw new IOException("Must create at least 3 regions"); + byte [] startKey = Bytes.toBytes("aaaaa"); + byte [] endKey = Bytes.toBytes("zzzzz"); + byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); + byte [][] regionStartKeys = new byte[splitKeys.length+1][]; + for (int i=0;i regions = getAllOnlineRegions(cluster); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // Add a new regionserver + log("Adding a fourth RS"); + RegionServerThread restarted = cluster.startRegionServer(); + expectedNumRS++; + restarted.waitForServerOnline(); + log("Additional RS is online"); + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // Master Restarts + List masterThreads = cluster.getMasterThreads(); + MasterThread activeMaster = null; + MasterThread backupMaster = null; + assertEquals(2, masterThreads.size()); + if (masterThreads.get(0).getMaster().isActiveMaster()) { + activeMaster = masterThreads.get(0); + backupMaster = masterThreads.get(1); + } else { + activeMaster = masterThreads.get(1); + backupMaster = masterThreads.get(0); + } + + // Bring down the backup master + LOG.debug("\n\nStopping backup master\n\n"); + backupMaster.getMaster().stop("Stop of backup during rolling restart"); + cluster.hbaseCluster.waitOnMaster(backupMaster); + + // Bring down the primary master + LOG.debug("\n\nStopping primary master\n\n"); + activeMaster.getMaster().stop("Stop of active during rolling restart"); + cluster.hbaseCluster.waitOnMaster(activeMaster); + + // Start primary master + LOG.debug("\n\nRestarting primary master\n\n"); + activeMaster = cluster.startMaster(); + cluster.waitForActiveAndReadyMaster(); + + // Start backup master + LOG.debug("\n\nRestarting backup master\n\n"); + backupMaster = cluster.startMaster(); + + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // RegionServer Restarts + + // Bring them down, one at a time, waiting between each to complete + List regionServers = + cluster.getLiveRegionServerThreads(); + int num = 1; + int total = regionServers.size(); + for (RegionServerThread rst : regionServers) { + String serverName = rst.getRegionServer().getServerName(); + log("Stopping region server " + num + " of " + total + " [ " + + serverName + "]"); + rst.getRegionServer().stop("Stopping RS during rolling restart"); + cluster.hbaseCluster.waitOnRegionServer(rst); + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, serverName); + log("RS shutdown done, waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + expectedNumRS--; + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + log("Restarting region server " + num + " of " + total); + restarted = cluster.startRegionServer(); + restarted.waitForServerOnline(); + expectedNumRS++; + log("Region server " + num + " is back online"); + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + num++; + } + Thread.sleep(2000); + assertRegionsAssigned(cluster, regions); + + // Bring the RS hosting ROOT down and the RS hosting META down at once + RegionServerThread rootServer = getServerHostingRoot(cluster); + RegionServerThread metaServer = getServerHostingMeta(cluster); + if (rootServer == metaServer) { + log("ROOT and META on the same server so killing another random server"); + int i=0; + while (rootServer == metaServer) { + metaServer = cluster.getRegionServerThreads().get(i); + i++; + } + } + log("Stopping server hosting ROOT"); + rootServer.getRegionServer().stop("Stopping ROOT server"); + log("Stopping server hosting META #1"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(rootServer); + log("Root server down"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down #1"); + expectedNumRS -= 2; + log("Waiting for meta server #1 RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // Kill off the server hosting META again + metaServer = getServerHostingMeta(cluster); + log("Stopping server hosting META #2"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down"); + expectedNumRS--; + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("RS shutdown done, waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + assertEquals(expectedNumRS, cluster.getRegionServerThreads().size()); + + // Start 3 RS again + cluster.startRegionServer().waitForServerOnline(); + cluster.startRegionServer().waitForServerOnline(); + cluster.startRegionServer().waitForServerOnline(); + Thread.sleep(1000); + log("Waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + // Shutdown server hosting META + metaServer = getServerHostingMeta(cluster); + log("Stopping server hosting META (1 of 3)"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down (1 of 3)"); + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("RS shutdown done, waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + + // Shutdown server hosting META again + metaServer = getServerHostingMeta(cluster); + log("Stopping server hosting META (2 of 3)"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down (2 of 3)"); + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("RS shutdown done, waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + + // Shutdown server hosting META again + metaServer = getServerHostingMeta(cluster); + log("Stopping server hosting META (3 of 3)"); + metaServer.getRegionServer().stop("Stopping META server"); + cluster.hbaseCluster.waitOnRegionServer(metaServer); + log("Meta server down (3 of 3)"); + log("Waiting for RS shutdown to be handled by master"); + waitForRSShutdownToStartAndFinish(activeMaster, + metaServer.getRegionServer().getServerName()); + log("RS shutdown done, waiting for no more RIT"); + ZKAssign.blockUntilNoRIT(zkw); + log("Verifying there are " + numRegions + " assigned on cluster"); + assertRegionsAssigned(cluster, regions); + + if (cluster.getRegionServerThreads().size() != 1) { + log("Online regionservers:"); + for (RegionServerThread rst : cluster.getRegionServerThreads()) { + log("RS: " + rst.getRegionServer().getServerName()); + } + } + assertEquals(1, cluster.getRegionServerThreads().size()); + + + // TODO: Bring random 3 of 4 RS down at the same time + + + // Stop the cluster + TEST_UTIL.shutdownMiniCluster(); + } + + private void waitForRSShutdownToStartAndFinish(MasterThread activeMaster, + String serverName) throws InterruptedException { + ServerManager sm = activeMaster.getMaster().getServerManager(); + // First wait for it to be in dead list + while (!sm.deadservers.isDeadServer(serverName)) { + log("Waiting for [" + serverName + "] to be listed as dead in master"); + Thread.sleep(100); + } + log("Server [" + serverName + "] marked as dead, waiting for it to " + + "finish dead processing"); + while (sm.deadservers.isDeadServer(serverName)) { + log("Server [" + serverName + "] still marked as dead, waiting"); + Thread.sleep(100); + } + log("Server [" + serverName + "] done with server shutdown processing"); + } + + private void log(String msg) { + LOG.debug("\n\n" + msg + "\n"); + } + + private RegionServerThread getServerHostingMeta(MiniHBaseCluster cluster) { + return getServerHosting(cluster, HRegionInfo.FIRST_META_REGIONINFO); + } + + private RegionServerThread getServerHostingRoot(MiniHBaseCluster cluster) { + return getServerHosting(cluster, HRegionInfo.ROOT_REGIONINFO); + } + + private RegionServerThread getServerHosting(MiniHBaseCluster cluster, + HRegionInfo region) { + for (RegionServerThread rst : cluster.getRegionServerThreads()) { + if (rst.getRegionServer().getOnlineRegions().contains(region)) { + return rst; + } + } + return null; + } + + private void assertRegionsAssigned(MiniHBaseCluster cluster, + Set expectedRegions) { + int numFound = 0; + for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) { + numFound += rst.getRegionServer().getNumberOfOnlineRegions(); + } + if (expectedRegions.size() != numFound) { + LOG.debug("Expected to find " + expectedRegions.size() + " but only found" + + " " + numFound); + NavigableSet foundRegions = getAllOnlineRegions(cluster); + for (String region : expectedRegions) { + if (!foundRegions.contains(region)) { + LOG.debug("Missing region: " + region); + } + } + assertEquals(expectedRegions.size(), numFound); + } else { + log("Success! Found expected number of " + numFound + " regions"); + } + } + + private NavigableSet getAllOnlineRegions(MiniHBaseCluster cluster) { + NavigableSet online = new TreeSet(); + for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) { + for (HRegionInfo region : rst.getRegionServer().getOnlineRegions()) { + online.add(region.getRegionNameAsString()); + } + } + return online; + } + +}