Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-8960: TestDistributedLogSplitting.testLogReplayForDisablingTable fails sometimes - Addendum

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1508316 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
jeffreyz 2013-07-30 06:27:18 +00:00
parent e36fc14007
commit 4a6de45f76
1 changed files with 87 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

236
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
View File

@ -233,37 +233,14 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
List<HRegionInfo> regions = null; HRegionServer hrs = findRSToKill(false, "table");
HRegionServer hrs = null; List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
for (int i = 0; i < NUM_RS; i++) {
boolean isCarryingMeta = false;
hrs = rsts.get(i).getRegionServer();
regions = ProtobufUtil.getOnlineRegions(hrs);
for (HRegionInfo region : regions) {
if (region.isMetaRegion()) {
isCarryingMeta = true;
break;
}
}
if (isCarryingMeta) {
continue;
}
break;
}
LOG.info("#regions = " + regions.size());
Iterator<HRegionInfo> it = regions.iterator();
while (it.hasNext()) {
HRegionInfo region = it.next();
if (region.isMetaTable()) {
it.remove();
}
}
makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
// wait for abort completes // wait for abort completes
this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
ht.close(); ht.close();
zkw.close();
} }
@Test(timeout = 300000) @Test(timeout = 300000)
@ -282,36 +259,13 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
List<HRegionInfo> regions = null; HRegionServer hrs = findRSToKill(true, "table");
HRegionServer hrs = null; List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
for (int i = 0; i < NUM_RS; i++) {
boolean isCarryingMeta = false;
hrs = rsts.get(i).getRegionServer();
regions = ProtobufUtil.getOnlineRegions(hrs);
for (HRegionInfo region : regions) {
if (region.isMetaRegion()) {
isCarryingMeta = true;
break;
}
}
if (!isCarryingMeta) {
continue;
}
break;
}
LOG.info("#regions = " + regions.size());
Iterator<HRegionInfo> it = regions.iterator();
while (it.hasNext()) {
HRegionInfo region = it.next();
if (region.isMetaTable()) {
it.remove();
}
}
makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
ht.close(); ht.close();
zkw.close();
} }
private void abortRSAndVerifyRecovery(HRegionServer hrs, HTable ht, final ZooKeeperWatcher zkw, private void abortRSAndVerifyRecovery(HRegionServer hrs, HTable ht, final ZooKeeperWatcher zkw,
@ -374,32 +328,8 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
List<HRegionInfo> regions = null; HRegionServer hrs = findRSToKill(false, "table");
HRegionServer hrs = null; List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
for (int i = 0; i < NUM_RS; i++) {
boolean isCarryingMeta = false;
hrs = rsts.get(i).getRegionServer();
regions = ProtobufUtil.getOnlineRegions(hrs);
for (HRegionInfo region : regions) {
if (region.isMetaRegion()) {
isCarryingMeta = true;
break;
}
}
if (isCarryingMeta) {
continue;
}
break;
}
LOG.info("#regions = " + regions.size());
Iterator<HRegionInfo> it = regions.iterator();
while (it.hasNext()) {
HRegionInfo region = it.next();
if (region.isMetaTable()) {
it.remove();
}
}
makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
// abort master // abort master
@ -436,6 +366,7 @@ public class TestDistributedLogSplitting {
assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
ht.close(); ht.close();
zkw.close();
} }
@Test(timeout = 300000) @Test(timeout = 300000)
@ -456,32 +387,8 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
List<HRegionInfo> regions = null; HRegionServer hrs = findRSToKill(false, "table");
HRegionServer hrs = null; List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
for (int i = 0; i < NUM_RS; i++) {
boolean isCarryingMeta = false;
hrs = rsts.get(i).getRegionServer();
regions = ProtobufUtil.getOnlineRegions(hrs);
for (HRegionInfo region : regions) {
if (region.isMetaRegion()) {
isCarryingMeta = true;
break;
}
}
if (isCarryingMeta) {
continue;
}
break;
}
LOG.info("#regions = " + regions.size());
Iterator<HRegionInfo> it = regions.iterator();
while (it.hasNext()) {
HRegionInfo region = it.next();
if (region.isMetaTable()) {
it.remove();
}
}
makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
// abort master // abort master
@ -520,6 +427,7 @@ public class TestDistributedLogSplitting {
assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
ht.close(); ht.close();
zkw.close();
} }
@ -601,6 +509,7 @@ public class TestDistributedLogSplitting {
assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
ht.close(); ht.close();
zkw.close();
} }
@Test(timeout = 300000) @Test(timeout = 300000)
@ -648,6 +557,7 @@ public class TestDistributedLogSplitting {
} }
}); });
ht.close(); ht.close();
zkw.close();
} }
@Test(timeout = 300000) @Test(timeout = 300000)
@ -680,7 +590,7 @@ public class TestDistributedLogSplitting {
if (isCarryingMeta) { if (isCarryingMeta) {
continue; continue;
} }
break; if (regions.size() > 0) break;
} }
this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1")); this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1"));
@ -693,6 +603,7 @@ public class TestDistributedLogSplitting {
TEST_UTIL.checksumRows(ht)); TEST_UTIL.checksumRows(ht));
ht.close(); ht.close();
zkw.close();
} }
@Test(timeout = 300000) @Test(timeout = 300000)
@ -825,6 +736,7 @@ public class TestDistributedLogSplitting {
} }
disablingHT.close(); disablingHT.close();
ht.close(); ht.close();
zkw.close();
} }
@Test(timeout = 300000) @Test(timeout = 300000)
@ -845,32 +757,8 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
List<HRegionInfo> regions = null; HRegionServer hrs = findRSToKill(false, "table");
HRegionServer hrs = null; List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
for (int i = 0; i < NUM_RS; i++) {
boolean isCarryingMeta = false;
hrs = rsts.get(i).getRegionServer();
regions = ProtobufUtil.getOnlineRegions(hrs);
for (HRegionInfo region : regions) {
if (region.isMetaRegion()) {
isCarryingMeta = true;
break;
}
}
if (isCarryingMeta) {
continue;
}
break;
}
LOG.info("#regions = " + regions.size());
Iterator<HRegionInfo> it = regions.iterator();
while (it.hasNext()) {
HRegionInfo region = it.next();
if (region.isMetaTable()) {
it.remove();
}
}
makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100); makeHLog(hrs.getWAL(), regions, "table", "family", NUM_LOG_LINES, 100);
// abort RS // abort RS
@ -910,6 +798,7 @@ public class TestDistributedLogSplitting {
} }
ht.close(); ht.close();
zkw.close();
} }
/** /**
@ -1028,6 +917,7 @@ public class TestDistributedLogSplitting {
assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION, assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION,
TEST_UTIL.countRows(ht)); TEST_UTIL.countRows(ht));
ht.close(); ht.close();
zkw.close();
} }
@ -1109,24 +999,9 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = new ZooKeeperWatcher(curConf, "table-creation", null); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(curConf, "table-creation", null);
List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads(); List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
installTable(zkw, "table", "family", 40); // only testing meta recovery in ZK operation
List<HRegionInfo> regions = null; HRegionServer hrs = findRSToKill(true, null);
HRegionServer hrs = null; List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs);
for (int i = 0; i < NUM_RS; i++) {
boolean isCarryingMeta = false;
hrs = rsts.get(i).getRegionServer();
regions = ProtobufUtil.getOnlineRegions(hrs);
for (HRegionInfo region : regions) {
if (region.isMetaRegion()) {
isCarryingMeta = true;
break;
}
}
if (!isCarryingMeta) {
continue;
}
break;
}
LOG.info("#regions = " + regions.size()); LOG.info("#regions = " + regions.size());
Set<HRegionInfo> tmpRegions = new HashSet<HRegionInfo>(); Set<HRegionInfo> tmpRegions = new HashSet<HRegionInfo>();
@ -1159,6 +1034,7 @@ public class TestDistributedLogSplitting {
} }
// meta region should be recovered // meta region should be recovered
assertFalse(isMetaRegionInRecovery); assertFalse(isMetaRegionInRecovery);
zkw.close();
} }
HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception { HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception {
@ -1242,6 +1118,7 @@ public class TestDistributedLogSplitting {
} }
hris.add(region); hris.add(region);
} }
LOG.info("Creating wal edits across " + hris.size() + " regions.");
for (int i = 0; i < edit_size; i++) { for (int i = 0; i < edit_size; i++) {
value[i] = (byte) ('a' + (i % 26)); value[i] = (byte) ('a' + (i % 26));
} }
@ -1386,4 +1263,65 @@ public class TestDistributedLogSplitting {
Thread.sleep(100); Thread.sleep(100);
} }
} }
/**
* Find a RS that has regions of a table.
* @param hasMetaRegion when true, the returned RS has META region as well
* @param tableName
* @return
* @throws Exception
*/
private HRegionServer findRSToKill(boolean hasMetaRegion, String tableName) throws Exception {
List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
int numOfRSs = rsts.size();
List<HRegionInfo> regions = null;
HRegionServer hrs = null;
for (int i = 0; i < numOfRSs; i++) {
boolean isCarryingMeta = false;
boolean foundTableRegion = false;
hrs = rsts.get(i).getRegionServer();
regions = ProtobufUtil.getOnlineRegions(hrs);
for (HRegionInfo region : regions) {
if (region.isMetaRegion()) {
isCarryingMeta = true;
}
if (tableName == null || region.getTableNameAsString().equalsIgnoreCase(tableName)) {
foundTableRegion = true;
}
if (foundTableRegion && (isCarryingMeta || !hasMetaRegion)) {
break;
}
}
if (isCarryingMeta && hasMetaRegion) {
// clients ask for a RS with META
if (!foundTableRegion) {
final HRegionServer destRS = hrs;
// the RS doesn't have regions of the specified table so we need move one to this RS
List<HRegionInfo> tableRegions =
TEST_UTIL.getHBaseAdmin().getTableRegions(Bytes.toBytes(tableName));
final HRegionInfo hri = tableRegions.get(0);
TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
Bytes.toBytes(destRS.getServerName().getServerName()));
// wait for region move completes
final RegionStates regionStates =
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
ServerName sn = regionStates.getRegionServerOfRegion(hri);
return (sn != null && sn.equals(destRS.getServerName()));
}
});
}
return hrs;
} else if (hasMetaRegion || isCarryingMeta) {
continue;
}
if (foundTableRegion) break;
}
return hrs;
}
} }