From 4a6de45f7655363fc8a0bf656d282a0ae1c73aff Mon Sep 17 00:00:00 2001 From: jeffreyz Date: Tue, 30 Jul 2013 06:27:18 +0000 Subject: [PATCH] HBASE-8960: TestDistributedLogSplitting.testLogReplayForDisablingTable fails sometimes - Addendum git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1508316 13f79535-47bb-0310-9956-ffa450edef68 --- .../master/TestDistributedLogSplitting.java | 236 +++++++----------- 1 file changed, 87 insertions(+), 149 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index c007627e6b1..a576e5843a7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -233,37 +233,14 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (isCarryingMeta) { - continue; - } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.isMetaTable()) { - it.remove(); - } - } + HRegionServer hrs = findRSToKill(false, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); // wait for abort completes this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); ht.close(); + zkw.close(); } @Test(timeout = 300000) @@ -282,36 +259,13 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (!isCarryingMeta) { - continue; - } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.isMetaTable()) { - it.remove(); - } - } + HRegionServer hrs = findRSToKill(true, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); ht.close(); + zkw.close(); } private void abortRSAndVerifyRecovery(HRegionServer hrs, HTable ht, final ZooKeeperWatcher zkw, @@ -374,32 +328,8 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (isCarryingMeta) { - continue; - } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.isMetaTable()) { - it.remove(); - } - } + HRegionServer hrs = findRSToKill(false, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); // abort master @@ -436,6 +366,7 @@ public class TestDistributedLogSplitting { assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); ht.close(); + zkw.close(); } @Test(timeout = 300000) @@ -456,32 +387,8 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (isCarryingMeta) { - continue; - } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.isMetaTable()) { - it.remove(); - } - } + HRegionServer hrs = findRSToKill(false, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); // abort master @@ -520,6 +427,7 @@ public class TestDistributedLogSplitting { assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); ht.close(); + zkw.close(); } @@ -601,6 +509,7 @@ public class TestDistributedLogSplitting { assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); ht.close(); + zkw.close(); } @Test(timeout = 300000) @@ -648,6 +557,7 @@ public class TestDistributedLogSplitting { } }); ht.close(); + zkw.close(); } @Test(timeout = 300000) @@ -680,7 +590,7 @@ public class TestDistributedLogSplitting { if (isCarryingMeta) { continue; } - break; + if (regions.size() > 0) break; } this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1")); @@ -693,6 +603,7 @@ public class TestDistributedLogSplitting { TEST_UTIL.checksumRows(ht)); ht.close(); + zkw.close(); } @Test(timeout = 300000) @@ -825,6 +736,7 @@ public class TestDistributedLogSplitting { } disablingHT.close(); ht.close(); + zkw.close(); } @Test(timeout = 300000) @@ -845,32 +757,8 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (isCarryingMeta) { - continue; - } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.isMetaTable()) { - it.remove(); - } - } + HRegionServer hrs = findRSToKill(false, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); // abort RS @@ -910,6 +798,7 @@ public class TestDistributedLogSplitting { } ht.close(); + zkw.close(); } /** @@ -1028,6 +917,7 @@ public class TestDistributedLogSplitting { assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION, TEST_UTIL.countRows(ht)); ht.close(); + zkw.close(); } @@ -1109,24 +999,9 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(curConf, "table-creation", null); List rsts = cluster.getLiveRegionServerThreads(); - installTable(zkw, "table", "family", 40); - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (!isCarryingMeta) { - continue; - } - break; - } + // only testing meta recovery in ZK operation + HRegionServer hrs = findRSToKill(true, null); + List regions = ProtobufUtil.getOnlineRegions(hrs); LOG.info("#regions = " + regions.size()); Set tmpRegions = new HashSet(); @@ -1159,6 +1034,7 @@ public class TestDistributedLogSplitting { } // meta region should be recovered assertFalse(isMetaRegionInRecovery); + zkw.close(); } HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception { @@ -1242,6 +1118,7 @@ public class TestDistributedLogSplitting { } hris.add(region); } + LOG.info("Creating wal edits across " + hris.size() + " regions."); for (int i = 0; i < edit_size; i++) { value[i] = (byte) ('a' + (i % 26)); } @@ -1386,4 +1263,65 @@ public class TestDistributedLogSplitting { Thread.sleep(100); } } + + /** + * Find a RS that has regions of a table. + * @param hasMetaRegion when true, the returned RS has META region as well + * @param tableName + * @return + * @throws Exception + */ + private HRegionServer findRSToKill(boolean hasMetaRegion, String tableName) throws Exception { + List rsts = cluster.getLiveRegionServerThreads(); + int numOfRSs = rsts.size(); + List regions = null; + HRegionServer hrs = null; + + for (int i = 0; i < numOfRSs; i++) { + boolean isCarryingMeta = false; + boolean foundTableRegion = false; + hrs = rsts.get(i).getRegionServer(); + regions = ProtobufUtil.getOnlineRegions(hrs); + for (HRegionInfo region : regions) { + if (region.isMetaRegion()) { + isCarryingMeta = true; + } + if (tableName == null || region.getTableNameAsString().equalsIgnoreCase(tableName)) { + foundTableRegion = true; + } + if (foundTableRegion && (isCarryingMeta || !hasMetaRegion)) { + break; + } + } + if (isCarryingMeta && hasMetaRegion) { + // clients ask for a RS with META + if (!foundTableRegion) { + final HRegionServer destRS = hrs; + // the RS doesn't have regions of the specified table so we need move one to this RS + List tableRegions = + TEST_UTIL.getHBaseAdmin().getTableRegions(Bytes.toBytes(tableName)); + final HRegionInfo hri = tableRegions.get(0); + TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(), + Bytes.toBytes(destRS.getServerName().getServerName())); + // wait for region move completes + final RegionStates regionStates = + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); + TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + ServerName sn = regionStates.getRegionServerOfRegion(hri); + return (sn != null && sn.equals(destRS.getServerName())); + } + }); + } + return hrs; + } else if (hasMetaRegion || isCarryingMeta) { + continue; + } + if (foundTableRegion) break; + } + + return hrs; + } + }