From 7a84ee262f5a0630009d3a9f9157e64aa1e910f2 Mon Sep 17 00:00:00 2001 From: Matteo Bertozzi Date: Fri, 20 May 2016 06:18:34 -0700 Subject: [PATCH] HBASE-15864 Reuse the testing helper to wait regions in transition --- .../hadoop/hbase/HBaseTestingUtility.java | 13 ++++++-- .../hadoop/hbase/TestRegionRebalancing.java | 21 +++++------- .../apache/hadoop/hbase/client/TestHCM.java | 18 +++------- .../hbase/coprocessor/TestMasterObserver.java | 21 ++---------- .../mapreduce/TestHFileOutputFormat2.java | 7 ++-- .../handler/TestTableDeleteFamilyHandler.java | 13 ++------ .../TestSplitTransactionOnCluster.java | 33 +++---------------- .../hadoop/hbase/util/TestHBaseFsckOneRS.java | 3 +- .../util/hbck/TestOfflineMetaRebuildHole.java | 9 ----- .../hbck/TestOfflineMetaRebuildOverlap.java | 8 ----- 10 files changed, 37 insertions(+), 109 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 501075a52d0..59b5bb33b4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -3887,13 +3887,20 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { /** * Wait until no regions in transition. * @param timeout How long to wait. - * @throws Exception + * @throws IOException */ - public void waitUntilNoRegionsInTransition( - final long timeout) throws Exception { + public void waitUntilNoRegionsInTransition(final long timeout) throws IOException { waitFor(timeout, predicateNoRegionsInTransition()); } + /** + * Wait until no regions in transition. (time limit 15min) + * @throws IOException + */ + public void waitUntilNoRegionsInTransition() throws IOException { + waitUntilNoRegionsInTransition(15 * 60000); + } + /** * Wait until labels is ready in VisibilityLabelsCache. * @param timeoutMillis diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java index 4c4697db52c..532896aec7c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java @@ -103,39 +103,39 @@ public class TestRegionRebalancing { admin.createTable(this.desc, Arrays.copyOfRange(HBaseTestingUtility.KEYS, 1, HBaseTestingUtility.KEYS.length)); this.regionLocator = connection.getRegionLocator(this.desc.getTableName()); - + MetaTableAccessor.fullScanMetaAndPrint(admin.getConnection()); - + assertEquals("Test table should have right number of regions", HBaseTestingUtility.KEYS.length, this.regionLocator.getStartKeys().length); - + // verify that the region assignments are balanced to start out assertRegionsAreBalanced(); - + // add a region server - total of 2 LOG.info("Started second server=" + UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName()); UTIL.getHBaseCluster().getMaster().balance(); assertRegionsAreBalanced(); - + // On a balanced cluster, calling balance() should return true assert(UTIL.getHBaseCluster().getMaster().balance() == true); - + // if we add a server, then the balance() call should return true // add a region server - total of 3 LOG.info("Started third server=" + UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName()); assert(UTIL.getHBaseCluster().getMaster().balance() == true); assertRegionsAreBalanced(); - + // kill a region server - total of 2 LOG.info("Stopped third server=" + UTIL.getHBaseCluster().stopRegionServer(2, false)); UTIL.getHBaseCluster().waitOnRegionServer(2); waitOnCrashProcessing(); UTIL.getHBaseCluster().getMaster().balance(); assertRegionsAreBalanced(); - + // start two more region servers - total of 4 LOG.info("Readding third server=" + UTIL.getHBaseCluster().startRegionServer().getRegionServer().getServerName()); @@ -253,10 +253,7 @@ public class TestRegionRebalancing { Thread.sleep(200); } catch (InterruptedException e) {} } - RegionStates regionStates = UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); - while (!regionStates.getRegionsInTransition().isEmpty()) { - Threads.sleep(100); - } + UTIL.waitUntilNoRegionsInTransition(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java index 289e6f25d45..8350b60b025 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java @@ -250,20 +250,14 @@ public class TestHCM { Table t = TEST_UTIL.createTable(tn, cf); TEST_UTIL.waitTableAvailable(tn); + TEST_UTIL.waitUntilNoRegionsInTransition(); - while(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). - getRegionStates().isRegionsInTransition()){ - Thread.sleep(1); - } final ConnectionImplementation hci = (ConnectionImplementation)TEST_UTIL.getConnection(); try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tn)) { while (l.getRegionLocation(rk).getPort() != sn.getPort()) { TEST_UTIL.getHBaseAdmin().move(l.getRegionLocation(rk).getRegionInfo(). getEncodedNameAsBytes(), Bytes.toBytes(sn.toString())); - while (TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). - getRegionStates().isRegionsInTransition()) { - Thread.sleep(1); - } + TEST_UTIL.waitUntilNoRegionsInTransition(); hci.clearRegionCache(tn); } Assert.assertNotNull(hci.clusterStatusListener); @@ -741,9 +735,7 @@ public class TestHCM { HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); // We can wait for all regions to be online, that makes log reading easier when debugging - while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { - Thread.sleep(1); - } + TEST_UTIL.waitUntilNoRegionsInTransition(); // Now moving the region to the second server HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation(); @@ -1034,9 +1026,7 @@ public class TestHCM { HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); // We can wait for all regions to be online, that makes log reading easier when debugging - while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { - Thread.sleep(1); - } + TEST_UTIL.waitUntilNoRegionsInTransition(); Put put = new Put(ROW_X); put.addColumn(FAM_NAM, ROW_X, ROW_X); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java index e1c4a1f56f3..b4e93bd8ec5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java @@ -51,11 +51,9 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.RegionPlan; -import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -1915,17 +1913,13 @@ public class TestMasterObserver { master.balanceSwitch(false); // wait for assignments to finish, if any - AssignmentManager mgr = master.getAssignmentManager(); - Set transRegions = mgr.getRegionStates().getRegionsInTransition(); - for (RegionState state : transRegions) { - mgr.getRegionStates().waitOnRegionToClearRegionsInTransition(state.getRegion()); - } + UTIL.waitUntilNoRegionsInTransition(); // move half the open regions from RS 0 to RS 1 HRegionServer rs = cluster.getRegionServer(0); byte[] destRS = Bytes.toBytes(cluster.getRegionServer(1).getServerName().toString()); //Make sure no regions are in transition now - waitForRITtoBeZero(master); + UTIL.waitUntilNoRegionsInTransition(); List openRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); int moveCnt = openRegions.size()/2; for (int i=0; i transRegions = mgr.getRegionStates().getRegionsInTransition(); - for (RegionState state : transRegions) { - mgr.getRegionStates().waitOnRegionToClearRegionsInTransition(state.getRegion()); - } - } - @Test (timeout=180000) public void testTableDescriptorsEnumeration() throws Exception { MiniHBaseCluster cluster = UTIL.getHBaseCluster(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index 4d92b8a5033..59173ad520f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -515,11 +515,8 @@ public class TestHFileOutputFormat2 { if (shouldChangeRegions) { LOG.info("Changing regions in table"); admin.disableTable(table.getName()); - while(util.getMiniHBaseCluster().getMaster().getAssignmentManager(). - getRegionStates().isRegionsInTransition()) { - Threads.sleep(200); - LOG.info("Waiting on table to finish disabling"); - } + util.waitUntilNoRegionsInTransition(); + util.deleteTable(table.getName()); byte[][] newSplitKeys = generateRandomSplitKeys(14); table = util.createTable(TABLE_NAME, FAMILIES, newSplitKeys); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java index 8a8c059ef67..aa7c5e1de35 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestTableDeleteFamilyHandler.java @@ -61,7 +61,7 @@ public class TestTableDeleteFamilyHandler { /** * Start up a mini cluster and put a small table of empty regions into it. - * + * * @throws Exception */ @BeforeClass @@ -80,15 +80,8 @@ public class TestTableDeleteFamilyHandler { // Create a table of three families. This will assign a region. TEST_UTIL.createTable(TABLENAME, FAMILIES); Table t = TEST_UTIL.getConnection().getTable(TABLENAME); - while(TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() - .getRegionStates().getRegionsInTransition().size() > 0) { - Thread.sleep(100); - } - // Create multiple regions in all the three column families - while(TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() - .getRegionStates().getRegionsInTransition().size() > 0) { - Thread.sleep(100); - } + TEST_UTIL.waitUntilNoRegionsInTransition(); + // Load the table with data for all families TEST_UTIL.loadTable(t, FAMILIES); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index e6578bb0f7d..7fbcfea84e4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -149,29 +149,12 @@ public class TestSplitTransactionOnCluster { this.admin.close(); } - private HRegionInfo getAndCheckSingleTableRegion(final List regions) { + private HRegionInfo getAndCheckSingleTableRegion(final List regions) + throws IOException, InterruptedException { assertEquals(1, regions.size()); HRegionInfo hri = regions.get(0).getRegionInfo(); - return waitOnRIT(hri); - } - - /** - * Often region has not yet fully opened. If we try to use it -- do a move for instance -- it - * will fail silently if the region is not yet opened. - * @param hri Region to check if in Regions In Transition... wait until out of transition before - * returning - * @return Passed in hri - */ - private HRegionInfo waitOnRIT(final HRegionInfo hri) { - // Close worked but we are going to open the region elsewhere. Before going on, make sure - // this completes. - while (TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). - getRegionStates().isRegionInTransition(hri)) { - LOG.info("Waiting on region in transition: " + - TESTING_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(). - getRegionTransitionState(hri)); - Threads.sleep(10); - } + TESTING_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .waitOnRegionToClearRegionsInTransition(hri, 600000); return hri; } @@ -210,13 +193,7 @@ public class TestSplitTransactionOnCluster { observer.latch.await(); LOG.info("Waiting for region to come out of RIT"); - TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates(); - return !regionStates.isRegionInTransition(hri.getEncodedName()); - } - }); + cluster.getMaster().getAssignmentManager().waitOnRegionToClearRegionsInTransition(hri, 60000); } finally { admin.setBalancerRunning(true, false); cluster.getMaster().setCatalogJanitorEnabled(true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java index 57bc77ed513..866a12dead3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java @@ -1683,8 +1683,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck { st.prepare(); st.stepsBeforePONR(regionServer, regionServer, false); AssignmentManager am = cluster.getMaster().getAssignmentManager(); - Set regionsInTransition = am.getRegionStates().getRegionsInTransition(); - for (RegionState state : regionsInTransition) { + for (RegionState state : am.getRegionStates().getRegionsInTransition()) { am.regionOffline(state.getRegion()); } Map regionsMap = new HashMap(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java index 615487d8c11..b8565e31c8a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java @@ -75,15 +75,6 @@ public class TestOfflineMetaRebuildHole extends OfflineMetaRebuildTestCore { LOG.info("Waiting for no more RIT"); TEST_UTIL.waitUntilNoRegionsInTransition(60000); - LOG.info("No more RIT in ZK, now doing final test verification"); - int tries = 60; - while(TEST_UTIL.getHBaseCluster() - .getMaster().getAssignmentManager().getRegionStates().getRegionsInTransition().size() > 0 && - tries-- > 0) { - LOG.info("Waiting for RIT: "+TEST_UTIL.getHBaseCluster() - .getMaster().getAssignmentManager().getRegionStates().getRegionsInTransition()); - Thread.sleep(1000); - } // Meta still messed up. assertEquals(1, scanMeta()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java index 40ba86a4d18..ae72935783a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java @@ -84,14 +84,6 @@ public class TestOfflineMetaRebuildOverlap extends OfflineMetaRebuildTestCore { LOG.info("Waiting for no more RIT"); TEST_UTIL.waitUntilNoRegionsInTransition(60000); LOG.info("No more RIT in ZK, now doing final test verification"); - int tries = 60; - while(TEST_UTIL.getHBaseCluster() - .getMaster().getAssignmentManager().getRegionStates().getRegionsInTransition().size() > 0 && - tries-- > 0) { - LOG.info("Waiting for RIT: "+TEST_UTIL.getHBaseCluster() - .getMaster().getAssignmentManager().getRegionStates().getRegionsInTransition()); - Thread.sleep(1000); - } // Meta still messed up. assertEquals(1, scanMeta());