From 05423b150bd609b1810c7dbc5173f8471d068bf7 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Tue, 13 Mar 2012 15:35:54 +0000 Subject: [PATCH] HBASE-5179 Handle potential data loss due to concurrent processing of processFaileOver and ServerShutdownHandler git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1300194 13f79535-47bb-0310-9956-ffa450edef68 --- .../hbase/master/AssignmentManager.java | 59 +++- .../apache/hadoop/hbase/master/HMaster.java | 77 ++++-- .../hadoop/hbase/master/MasterServices.java | 7 +- .../hadoop/hbase/master/ServerManager.java | 41 ++- .../master/handler/CreateTableHandler.java | 2 + .../hbase/regionserver/HRegionServer.java | 2 +- .../hbase/master/TestAssignmentManager.java | 20 +- .../hbase/master/TestCatalogJanitor.java | 17 +- .../TestRSKilledWhenMasterInitializing.java | 261 ++++++++++++++++++ 9 files changed, 439 insertions(+), 47 deletions(-) create mode 100644 src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index ae468ca4fbf..7784c75b421 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -65,7 +65,6 @@ import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.RegionTransitionData; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; -import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State; import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler; import org.apache.hadoop.hbase.master.handler.DisableTableHandler; @@ -328,11 +327,13 @@ public class AssignmentManager extends ZooKeeperListener { /** * Called on startup. * Figures whether a fresh cluster start of we are joining extant running cluster. + * @param onlineServers onlined servers when master started * @throws IOException * @throws KeeperException * @throws InterruptedException */ - void joinCluster() throws IOException, KeeperException, InterruptedException { + void joinCluster(final Set onlineServers) throws IOException, + KeeperException, InterruptedException { // Concurrency note: In the below the accesses on regionsInTransition are // outside of a synchronization block where usually all accesses to RIT are // synchronized. The presumption is that in this case it is safe since this @@ -343,7 +344,7 @@ public class AssignmentManager extends ZooKeeperListener { // Scan META to build list of existing regions, servers, and assignment // Returns servers who have not checked in (assumed dead) and their regions - Map>> deadServers = rebuildUserRegions(); + Map>> deadServers = rebuildUserRegions(onlineServers); processDeadServersAndRegionsInTransition(deadServers); @@ -353,6 +354,16 @@ public class AssignmentManager extends ZooKeeperListener { recoverTableInEnablingState(this.enablingTables, isWatcherCreated); } + /** + * Only used for tests + * @throws IOException + * @throws KeeperException + * @throws InterruptedException + */ + void joinCluster() throws IOException, KeeperException, InterruptedException { + joinCluster(serverManager.getOnlineServers().keySet()); + } + /** * Process all regions that are in transition up in zookeeper. Used by * master joining an already running cluster. @@ -398,6 +409,12 @@ public class AssignmentManager extends ZooKeeperListener { } } + // Remove regions in RIT, they are possibly being processed by + // ServerShutdownHandler. + synchronized (regionsInTransition) { + nodes.removeAll(regionsInTransition.keySet()); + } + // If we found user regions out on cluster, its a failover. if (this.failover) { LOG.info("Found regions out on cluster or in RIT; failover"); @@ -1770,6 +1787,7 @@ public class AssignmentManager extends ZooKeeperListener { final List servers = this.serverManager.getOnlineServersList(); final List drainingServers = this.serverManager.getDrainingServersList(); + if (serverToExclude != null) servers.remove(serverToExclude); // Loop through the draining server list and remove them from the server @@ -1782,6 +1800,11 @@ public class AssignmentManager extends ZooKeeperListener { } } + // Remove the deadNotExpired servers from the server list. + removeDeadNotExpiredServers(servers); + + + if (servers.isEmpty()) return null; RegionPlan randomPlan = new RegionPlan(state.getRegion(), null, @@ -1813,7 +1836,7 @@ public class AssignmentManager extends ZooKeeperListener { " so generated a random one; " + randomPlan + "; " + serverManager.countOfRegionServers() + " (online=" + serverManager.getOnlineServers().size() + - ", exclude=" + drainingServers.size() + ") available servers"); + ", available=" + servers.size() + ") available servers"); return randomPlan; } LOG.debug("Using pre-existing plan for region " + @@ -1821,6 +1844,23 @@ public class AssignmentManager extends ZooKeeperListener { return existingPlan; } + /** + * Loop through the deadNotExpired server list and remove them from the + * servers. + * @param servers + */ + public void removeDeadNotExpiredServers(List servers) { + Set deadNotExpiredServers = this.serverManager + .getDeadNotExpiredServers(); + if (!deadNotExpiredServers.isEmpty()) { + for (ServerName server : deadNotExpiredServers) { + LOG.debug("Removing dead but not expired server: " + server + + " from eligible server pool."); + servers.remove(server); + } + } + } + /** * Unassign the list of regions. Configuration knobs: * hbase.bulk.waitbetween.reopen indicates the number of milliseconds to @@ -2134,6 +2174,7 @@ public class AssignmentManager extends ZooKeeperListener { throws IOException, InterruptedException { List servers = this.serverManager.getOnlineServersList(); + removeDeadNotExpiredServers(servers); assignUserRegions(regions, servers); } @@ -2173,6 +2214,9 @@ public class AssignmentManager extends ZooKeeperListener { // Get all available servers List servers = serverManager.getOnlineServersList(); + // Remove the deadNotExpired servers from the server list. + removeDeadNotExpiredServers(servers); + // If there are no servers we need not proceed with region assignment. if(servers.isEmpty()) return; @@ -2377,11 +2421,14 @@ public class AssignmentManager extends ZooKeeperListener { *

* Returns a map of servers that are not found to be online and the regions * they were hosting. + * @param onlineServers if one region's location belongs to onlineServers, it + * doesn't need to be assigned. * @return map of servers not online to their assigned regions, as stored * in META * @throws IOException */ - Map>> rebuildUserRegions() + Map>> rebuildUserRegions( + final Set onlineServers) throws IOException, KeeperException { // Region assignment from META List results = MetaReader.fullScan(this.catalogTracker); @@ -2414,7 +2461,7 @@ public class AssignmentManager extends ZooKeeperListener { } addTheTablesInPartialState(this.disablingTables, this.enablingTables, regionInfo, tableName); - } else if (!this.serverManager.isServerOnline(regionLocation)) { + } else if (!onlineServers.contains(regionLocation)) { // Region is located on a server that isn't online List> offlineRegions = offlineServers.get(regionLocation); diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index c4b4d3060dc..03623388a48 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -26,6 +26,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -191,6 +192,8 @@ Server { private volatile boolean isActiveMaster = false; // flag set after we complete initialization once active (used for testing) private volatile boolean initialized = false; + // flag set after we complete assignRootAndMeta. + private volatile boolean serverShutdownHandlerEnabled = false; // Instance of the hbase executor service. ExecutorService executorService; @@ -527,13 +530,17 @@ Server { } } + Set onlineServers = new HashSet(serverManager + .getOnlineServers().keySet()); // TODO: Should do this in background rather than block master startup status.setStatus("Splitting logs after master startup"); - this.fileSystemManager. - splitLogAfterStartup(this.serverManager.getOnlineServers().keySet()); + splitLogAfterStartup(this.fileSystemManager, onlineServers); // Make sure root and meta assigned before proceeding. assignRootAndMeta(status); + serverShutdownHandlerEnabled = true; + this.serverManager.expireDeadNotExpiredServers(); + // Update meta with new HRI if required. i.e migrate all HRI with HTD to // HRI with out HTD in meta and update the status in ROOT. This must happen // before we assign all user regions or else the assignment will fail. @@ -543,7 +550,7 @@ Server { // Fixup assignment manager status status.setStatus("Starting assignment manager"); - this.assignmentManager.joinCluster(); + this.assignmentManager.joinCluster(onlineServers); this.balancer.setClusterStatus(getClusterStatus()); this.balancer.setMasterServices(this); @@ -578,6 +585,16 @@ Server { } } + /** + * Override to change master's splitLogAfterStartup. Used testing + * @param mfs + * @param onlineServers + */ + protected void splitLogAfterStartup(final MasterFileSystem mfs, + Set onlineServers) { + mfs.splitLogAfterStartup(onlineServers); + } + /** * Check -ROOT- and .META. are assigned. If not, * assign them. @@ -595,17 +612,11 @@ Server { status.setStatus("Assigning ROOT region"); boolean rit = this.assignmentManager. processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.ROOT_REGIONINFO); - ServerName expiredServer = null; + ServerName currentRootServer = null; if (!catalogTracker.verifyRootRegionLocation(timeout)) { - ServerName currentRootServer = this.catalogTracker.getRootLocation(); - if (expireIfOnline(currentRootServer)) { - // We are expiring this server. The processing of expiration will assign - // root so don't do it here. - expiredServer = currentRootServer; - } else { - // Root was not on an online server when we failed verification - this.assignmentManager.assignRoot(); - } + currentRootServer = this.catalogTracker.getRootLocation(); + splitLogAndExpireIfOnline(currentRootServer); + this.assignmentManager.assignRoot(); this.catalogTracker.waitForRoot(); //This guarantees that the transition has completed this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO); @@ -625,13 +636,11 @@ Server { if (!this.catalogTracker.verifyMetaRegionLocation(timeout)) { ServerName currentMetaServer = this.catalogTracker.getMetaLocationOrReadLocationFromRoot(); - if (currentMetaServer != null && currentMetaServer.equals(expiredServer)) { - // We are expiring the server that is carrying meta already. - // The expiration processing will take care of reassigning meta. - expireIfOnline(currentMetaServer); - } else { - this.assignmentManager.assignMeta(); + if (currentMetaServer != null + && !currentMetaServer.equals(currentRootServer)) { + splitLogAndExpireIfOnline(currentMetaServer); } + assignmentManager.assignMeta(); this.catalogTracker.waitForMeta(); // Above check waits for general meta availability but this does not // guarantee that the transition has completed @@ -682,16 +691,19 @@ Server { } /** - * Expire a server if we find it is one of the online servers set. + * Split a server's log and expire it if we find it is one of the online + * servers. * @param sn ServerName to check. - * @return True if server was online and so we expired it as unreachable. + * @throws IOException */ - private boolean expireIfOnline(final ServerName sn) { - if (sn == null) return false; - if (!this.serverManager.isServerOnline(sn)) return false; - LOG.info("Forcing expiration of " + sn); - this.serverManager.expireServer(sn); - return true; + private void splitLogAndExpireIfOnline(final ServerName sn) + throws IOException { + if (sn == null || !serverManager.isServerOnline(sn)) { + return; + } + LOG.info("Forcing splitLog and expire of " + sn); + fileSystemManager.splitLog(sn); + serverManager.expireServer(sn); } @Override @@ -1692,7 +1704,16 @@ Server { public boolean isInitialized() { return initialized; } - + + /** + * ServerShutdownHandlerEnabled is set false before completing + * assignRootAndMeta to prevent processing of ServerShutdownHandler. + * @return true if assignRootAndMeta has completed; + */ + public boolean isServerShutdownHandlerEnabled() { + return this.serverShutdownHandlerEnabled; + } + @Override @Deprecated public void assign(final byte[] regionName, final boolean force) diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index fb6c7e56e7b..e1e6685891a 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -25,8 +25,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.TableDescriptors; -import org.apache.hadoop.hbase.TableNotDisabledException; -import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.zookeeper.MasterSchemaChangeTracker; @@ -94,4 +92,9 @@ public interface MasterServices extends Server { */ public RegionServerTracker getRegionServerTracker(); + /** + * @return true if master enables ServerShutdownHandler; + */ + public boolean isServerShutdownHandlerEnabled(); + } diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 6fb68796a36..cbd55f749c2 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -24,6 +24,8 @@ import java.net.InetAddress; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -46,10 +48,10 @@ import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.ipc.HRegionInterface; -import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler; import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler; import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; /** * The ServerManager class manages info about region servers. @@ -97,6 +99,14 @@ public class ServerManager { private final long maxSkew; + /** + * Set of region servers which are dead but not expired immediately. If one + * server died before master enables ServerShutdownHandler, the server will be + * added to set and will be expired through calling + * {@link ServerManager#expireDeadNotExpiredServers()} by master. + */ + private Set deadNotExpiredServers = new HashSet(); + /** * Constructor. * @param master @@ -349,6 +359,12 @@ public class ServerManager { * shutdown processing. */ public synchronized void expireServer(final ServerName serverName) { + if (!services.isServerShutdownHandlerEnabled()) { + LOG.info("Master doesn't enable ServerShutdownHandler during initialization, " + + "delay expiring server " + serverName); + this.deadNotExpiredServers.add(serverName); + return; + } excludeRegionServerFromSchemaChanges(serverName); if (!this.onlineServers.containsKey(serverName)) { LOG.warn("Received expiration of " + serverName + @@ -395,6 +411,22 @@ public class ServerManager { carryingRoot + ", meta=" + carryingMeta); } + /** + * Expire the servers which died during master's initialization. It will be + * called after HMaster#assignRootAndMeta. + * @throws IOException + * */ + synchronized void expireDeadNotExpiredServers() throws IOException { + if (!services.isServerShutdownHandlerEnabled()) { + throw new IOException("Master hasn't enabled ServerShutdownHandler "); + } + Iterator serverIterator = deadNotExpiredServers.iterator(); + while (serverIterator.hasNext()) { + expireServer(serverIterator.next()); + serverIterator.remove(); + } + } + /* * Remove the server from the drain list. */ @@ -606,6 +638,13 @@ public class ServerManager { return new ArrayList(this.drainingServers); } + /** + * @return A copy of the internal set of deadNotExpired servers. + */ + Set getDeadNotExpiredServers() { + return new HashSet(this.deadNotExpiredServers); + } + public boolean isServerOnline(ServerName serverName) { return onlineServers.containsKey(serverName); } diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java index bb57f8db4b2..d208b457be4 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java +++ b/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java @@ -172,6 +172,8 @@ public class CreateTableHandler extends EventHandler { // 4. Trigger immediate assignment of the regions in round-robin fashion List servers = serverManager.getOnlineServersList(); + // Remove the deadNotExpired servers from the server list. + assignmentManager.removeDeadNotExpiredServers(servers); try { this.assignmentManager.assignUserRegions(Arrays.asList(newRegions), servers); diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index a9a561040cb..e0af8fb2955 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -780,7 +780,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, // Interrupt catalog tracker here in case any regions being opened out in // handlers are stuck waiting on meta or root. if (this.catalogTracker != null) this.catalogTracker.stop(); - if (this.fsOk) { + if (!this.killed && this.fsOk) { waitOnAllRegionsToClose(abortRequested); LOG.info("stopping server " + this.serverNameFromMasterPOV + "; all regions closed."); diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java index d68ce332ac6..841649abb3f 100644 --- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java +++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java @@ -22,17 +22,22 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerLoad; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; @@ -45,6 +50,7 @@ import org.apache.hadoop.hbase.ipc.HRegionInterface; import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.hbase.zookeeper.ZKAssign; @@ -59,10 +65,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.client.Get; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; /** @@ -115,10 +117,12 @@ public class TestAssignmentManager { this.serverManager = Mockito.mock(ServerManager.class); Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true); - final List onlineServers = new ArrayList(); - onlineServers.add(SERVERNAME_B); - onlineServers.add(SERVERNAME_A); - Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(onlineServers); + final Map onlineServers = new HashMap(); + onlineServers.put(SERVERNAME_B, new HServerLoad()); + onlineServers.put(SERVERNAME_A, new HServerLoad()); + Mockito.when(this.serverManager.getOnlineServersList()).thenReturn( + new ArrayList(onlineServers.keySet())); + Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers); Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)). thenReturn(true); Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)). diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index a5628b95e2f..227c5f28446 100644 --- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -35,7 +35,17 @@ import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; @@ -260,6 +270,11 @@ public class TestCatalogJanitor { public RegionServerTracker getRegionServerTracker() { return null; } + + @Override + public boolean isServerShutdownHandlerEnabled() { + return true; + } } @Test diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java new file mode 100644 index 00000000000..8f8018dc77c --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenMasterInitializing.java @@ -0,0 +1,261 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.TestMasterFailover; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestRSKilledWhenMasterInitializing { + private static final Log LOG = LogFactory.getLog(TestMasterFailover.class); + + private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility(); + private static final int NUM_MASTERS = 1; + private static final int NUM_RS = 4; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set it so that this test runs with my custom master + TESTUTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, + TestingMaster.class, HMaster.class); + // Start up the cluster. + TESTUTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + if (!TESTUTIL.getHBaseCluster().getMaster().isInitialized()) { + // master is not initialized and is waiting something forever. + for (MasterThread mt : TESTUTIL.getHBaseCluster().getLiveMasterThreads()) { + mt.interrupt(); + } + } + TESTUTIL.shutdownMiniCluster(); + } + + /** + * An HMaster instance used in this test. If 'TestingMaster.sleep' is set in + * the Configuration, then we'll sleep after log is split and we'll also + * return a custom RegionServerTracker. + */ + public static class TestingMaster extends HMaster { + private boolean logSplit = false; + + public TestingMaster(Configuration conf) throws IOException, + KeeperException, InterruptedException { + super(conf); + } + + @Override + protected void splitLogAfterStartup(MasterFileSystem mfs, + Set onlineServers) { + super.splitLogAfterStartup(mfs, onlineServers); + logSplit = true; + // If "TestingMaster.sleep" is set, sleep after log split. + if (getConfiguration().getBoolean("TestingMaster.sleep", false)) { + int duration = getConfiguration().getInt( + "TestingMaster.sleep.duration", 0); + Threads.sleep(duration); + } + } + + + public boolean isLogSplitAfterStartup() { + return logSplit; + } + } + + @Test(timeout = 120000) + public void testCorrectnessWhenMasterFailOver() throws Exception { + final byte[] TABLENAME = Bytes.toBytes("testCorrectnessWhenMasterFailOver"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[][] SPLITKEYS = { Bytes.toBytes("b"), Bytes.toBytes("i") }; + + MiniHBaseCluster cluster = TESTUTIL.getHBaseCluster(); + + HTableDescriptor desc = new HTableDescriptor(TABLENAME); + desc.addFamily(new HColumnDescriptor(FAMILY)); + HBaseAdmin hbaseAdmin = TESTUTIL.getHBaseAdmin(); + hbaseAdmin.createTable(desc, SPLITKEYS); + + assertTrue(hbaseAdmin.isTableAvailable(TABLENAME)); + + HTable table = new HTable(TESTUTIL.getConfiguration(), TABLENAME); + List puts = new ArrayList(); + Put put1 = new Put(Bytes.toBytes("a")); + put1.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value")); + Put put2 = new Put(Bytes.toBytes("h")); + put2.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value")); + Put put3 = new Put(Bytes.toBytes("o")); + put3.add(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes("value")); + puts.add(put1); + puts.add(put2); + puts.add(put3); + table.put(puts); + ResultScanner resultScanner = table.getScanner(new Scan()); + int count = 0; + while (resultScanner.next() != null) { + count++; + } + resultScanner.close(); + table.close(); + assertEquals(3, count); + + /* Starting test */ + cluster.getConfiguration().setBoolean("TestingMaster.sleep", true); + cluster.getConfiguration().setInt("TestingMaster.sleep.duration", 10000); + + /* NO.1 .META. region correctness */ + // First abort master + abortMaster(cluster); + TestingMaster master = startMasterAndWaitUntilLogSplit(cluster); + + // Second kill meta server + int metaServerNum = cluster.getServerWithMeta(); + int rootServerNum = cluster.getServerWith(HRegionInfo.ROOT_REGIONINFO + .getRegionName()); + HRegionServer metaRS = cluster.getRegionServer(metaServerNum); + LOG.debug("Killing metaRS and carryingRoot = " + + (metaServerNum == rootServerNum)); + metaRS.kill(); + metaRS.join(); + + /* + * Sleep double time of TestingMaster.sleep.duration, so we can ensure that + * master has already assigned ROOTandMETA or is blocking on assigning + * ROOTandMETA + */ + Thread.sleep(10000 * 2); + + waitUntilMasterIsInitialized(master); + + // Third check whether data is correct in meta region + assertTrue(hbaseAdmin.isTableAvailable(TABLENAME)); + + /* + * NO.2 -ROOT- region correctness . If the .META. server killed in the NO.1 + * is also carrying -ROOT- region, it is not needed + */ + if (rootServerNum != metaServerNum) { + // First abort master + abortMaster(cluster); + master = startMasterAndWaitUntilLogSplit(cluster); + + // Second kill meta server + HRegionServer rootRS = cluster.getRegionServer(rootServerNum); + LOG.debug("Killing rootRS"); + rootRS.kill(); + rootRS.join(); + + /* + * Sleep double time of TestingMaster.sleep.duration, so we can ensure + * that master has already assigned ROOTandMETA or is blocking on + * assigning ROOTandMETA + */ + Thread.sleep(10000 * 2); + waitUntilMasterIsInitialized(master); + + // Third check whether data is correct in meta region + assertTrue(hbaseAdmin.isTableAvailable(TABLENAME)); + } + + /* NO.3 data region correctness */ + ServerManager serverManager = cluster.getMaster().getServerManager(); + while (serverManager.areDeadServersInProgress()) { + Thread.sleep(100); + } + table = new HTable(TESTUTIL.getConfiguration(), TABLENAME); + resultScanner = table.getScanner(new Scan()); + count = 0; + while (resultScanner.next() != null) { + count++; + } + resultScanner.close(); + table.close(); + assertEquals(3, count); + } + + private void abortMaster(MiniHBaseCluster cluster) + throws InterruptedException { + for (MasterThread mt : cluster.getLiveMasterThreads()) { + if (mt.getMaster().isActiveMaster()) { + mt.getMaster().abort("Aborting for tests", new Exception("Trace info")); + mt.join(); + break; + } + } + LOG.debug("Master is aborted"); + } + + private TestingMaster startMasterAndWaitUntilLogSplit(MiniHBaseCluster cluster) + throws IOException, InterruptedException { + TestingMaster master = (TestingMaster) cluster.startMaster().getMaster(); + while (!master.isLogSplitAfterStartup()) { + Thread.sleep(100); + } + LOG.debug("splitted:" + master.isLogSplitAfterStartup() + ",initialized:" + + master.isInitialized()); + return master; + } + + private void waitUntilMasterIsInitialized(HMaster master) + throws InterruptedException { + while (!master.isInitialized()) { + Thread.sleep(100); + } + LOG.debug("master isInitialized"); + } + + @org.junit.Rule + public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = + new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); + +}