diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index f66882da735..d35cebbb87f 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -582,8 +582,12 @@ public class AssignmentManager extends ZooKeeperListener { // So we will assign the ROOT and .META. region immediately. processOpeningState(regionInfo); break; - - } + } else if (deadServers.keySet().contains(sn)) { + // if the region is found on a dead server, we can assign + // it to a new RS. (HBASE-5882) + processOpeningState(regionInfo); + break; + } regionsInTransition.put(encodedRegionName, getRegionState(regionInfo, RegionState.State.OPENING, rt)); failoverProcessedRegions.put(encodedRegionName, regionInfo); @@ -621,7 +625,6 @@ public class AssignmentManager extends ZooKeeperListener { } } - /** * Put the region hri into an offline state up in zk. * @param hri diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java index 8b2cbec2694..aeb655c8344 100644 --- a/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java +++ b/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java @@ -472,6 +472,7 @@ public class TestAssignmentManager { am.regionsInTransition.isEmpty()); } } finally { + am.setEnabledTable(REGIONINFO.getTableNameAsString()); executor.shutdown(); am.shutdown(); // Clean up all znodes @@ -614,47 +615,65 @@ public class TestAssignmentManager { @Test public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException, ServiceException, InterruptedException { - this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, - MockedLoadBalancer.class, LoadBalancer.class); - AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, - this.serverManager); - // Boolean variable used for waiting until randomAssignment is called and new - // plan is generated. - AtomicBoolean gate = new AtomicBoolean(false); - if (balancer instanceof MockedLoadBalancer) { - ((MockedLoadBalancer) balancer).setGateVariable(gate); + try { + this.server.getConfiguration().setClass( + HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockedLoadBalancer.class, + LoadBalancer.class); + AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager( + this.server, this.serverManager); + // Boolean variable used for waiting until randomAssignment is called and + // new + // plan is generated. + AtomicBoolean gate = new AtomicBoolean(false); + if (balancer instanceof MockedLoadBalancer) { + ((MockedLoadBalancer) balancer).setGateVariable(gate); + } + ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A); + int v = ZKAssign.getVersion(this.watcher, REGIONINFO); + ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, + EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v); + String path = ZKAssign.getNodeName(this.watcher, REGIONINFO + .getEncodedName()); + RegionState state = new RegionState(REGIONINFO, State.OPENING, System + .currentTimeMillis(), SERVERNAME_A); + am.regionsInTransition.put(REGIONINFO.getEncodedName(), state); + // a dummy plan inserted into the regionPlans. This plan is cleared and + // new one is formed + am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan( + REGIONINFO, null, SERVERNAME_A)); + RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); + List serverList = new ArrayList(2); + serverList.add(SERVERNAME_B); + Mockito.when( + this.serverManager.createDestinationServersList(SERVERNAME_A)) + .thenReturn(serverList); + am.nodeDataChanged(path); + // here we are waiting until the random assignment in the load balancer is + // called. + while (!gate.get()) { + Thread.sleep(10); + } + // new region plan may take some time to get updated after random + // assignment is called and + // gate is set to true. + RegionPlan newRegionPlan = am.regionPlans + .get(REGIONINFO.getEncodedName()); + while (newRegionPlan == null) { + Thread.sleep(10); + newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); + } + // the new region plan created may contain the same RS as destination but + // it should + // be new plan. + assertNotSame("Same region plan should not come", regionPlan, + newRegionPlan); + assertTrue("Destnation servers should be different.", !(regionPlan + .getDestination().equals(newRegionPlan.getDestination()))); + } finally { + this.server.getConfiguration().setClass( + HConstants.HBASE_MASTER_LOADBALANCER_CLASS, DefaultLoadBalancer.class, + LoadBalancer.class); } - ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A); - int v = ZKAssign.getVersion(this.watcher, REGIONINFO); - ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, - EventType.RS_ZK_REGION_FAILED_OPEN, v); - String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName()); - RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(), - SERVERNAME_A); - am.regionsInTransition.put(REGIONINFO.getEncodedName(), state); - // a dummy plan inserted into the regionPlans. This plan is cleared and new one is formed - am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A)); - RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); - List serverList = new ArrayList(2); - serverList.add(SERVERNAME_B); - Mockito.when(this.serverManager.createDestinationServersList(SERVERNAME_A)).thenReturn(serverList); - am.nodeDataChanged(path); - // here we are waiting until the random assignment in the load balancer is called. - while (!gate.get()) { - Thread.sleep(10); - } - // new region plan may take some time to get updated after random assignment is called and - // gate is set to true. - RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); - while (newRegionPlan == null) { - Thread.sleep(10); - newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); - } - // the new region plan created may contain the same RS as destination but it should - // be new plan. - assertNotSame("Same region plan should not come", regionPlan, newRegionPlan); - assertTrue("Destnation servers should be different.", !(regionPlan.getDestination().equals( - newRegionPlan.getDestination()))); } /** @@ -676,6 +695,35 @@ public class TestAssignmentManager { } } + /** + * Test the scenario when the master is in failover and trying to process a + * region which is in Opening state on a dead RS. Master should immediately + * assign the region and not wait for Timeout Monitor.(Hbase-5882). + */ + @Test + public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException, + KeeperException, ServiceException, InterruptedException { + AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, + this.serverManager); + ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A); + int version = ZKAssign.getVersion(this.watcher, REGIONINFO); + ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, + EventType.RS_ZK_REGION_OPENING, version); + RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING, + REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY); + Map>> deadServers = + new HashMap>>(); + deadServers.put(SERVERNAME_A, null); + version = ZKAssign.getVersion(this.watcher, REGIONINFO); + am.gate.set(false); + am.processRegionsInTransition(rt, REGIONINFO, deadServers, version); + // Waiting for the assignment to get completed. + while (!am.gate.get()) { + Thread.sleep(10); + } + assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO + .getEncodedName())); + } /** * Creates a new ephemeral node in the SPLITTING state for the specified region. * Create it ephemeral in case regionserver dies mid-split. @@ -810,7 +858,13 @@ public class TestAssignmentManager { while (this.gate.get()) Threads.sleep(1); super.processRegionsInTransition(rt, regionInfo, deadServers, expectedVersion); } - + + @Override + public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan, + boolean hijack) { + super.assign(region, setOfflineInZK, forceNewPlan, hijack); + this.gate.set(true); + } /** reset the watcher */ void setWatcher(ZooKeeperWatcher watcher) { this.watcher = watcher;