From c127577862715425015d17aeb7952cf6b9f4ae22 Mon Sep 17 00:00:00 2001 From: stack Date: Tue, 25 Aug 2015 16:16:03 -0700 Subject: [PATCH] HBASE-14290 Spin up less threads in tests Conflicts: hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java --- .../apache/hadoop/hbase/ipc/AsyncCall.java | 5 +- .../hadoop/hbase/ipc/AsyncRpcChannel.java | 4 +- .../apache/hadoop/hbase/ipc/RpcServer.java | 2 +- .../client/TestCloneSnapshotFromClient.java | 1 - .../master/TestDistributedLogSplitting.java | 1464 +++++++++-------- .../hbase/master/TestRollingRestart.java | 3 +- .../TestRegionReplicaFailover.java | 1 - .../hbase/snapshot/TestExportSnapshot.java | 1 - .../src/test/resources/hbase-site.xml | 8 + 9 files changed, 780 insertions(+), 709 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java index ec1909a16cd..431c669c982 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java @@ -86,8 +86,9 @@ public class AsyncCall extends DefaultPromise { @Override public String toString() { - return "callId: " + this.id + " methodName: " + this.method.getName() + " param {" - + (this.param != null ? ProtobufUtil.getShortTextFormat(this.param) : "") + "}"; + return "callId=" + this.id + ", method=" + this.method.getName() + + ", rpcTimeout=" + this.rpcTimeout + ", param {" + + (this.param != null ? ProtobufUtil.getShortTextFormat(this.param) : "") + "}"; } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java index cfc8b1ba1b9..dbc39cf06a5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java @@ -146,10 +146,10 @@ public class AsyncRpcChannel { this.channel = connect(bootstrap).channel(); - name = ("IPC Client (" + channel.hashCode() + ") connection to " + + name = ("IPC Client (" + channel.hashCode() + ") to " + address.toString() + ((ticket == null) ? - " from an unknown user" : + " from unknown user" : (" from " + ticket.getName()))); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index b1aa66c9c64..0ee93eb2a4b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -595,7 +595,7 @@ public class RpcServer implements RpcServerInterface { readers[i] = reader; readPool.execute(reader); } - LOG.info(getName() + ": started " + readThreads + " reader(s)."); + LOG.info(getName() + ": started " + readThreads + " reader(s) listening on port=" + port); // Register accepts on the server socket with the selector. acceptChannel.register(selector, SelectionKey.OP_ACCEPT); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java index 19e7410fdb3..d8a1e47ec4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java @@ -70,7 +70,6 @@ public class TestCloneSnapshotFromClient { TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); TEST_UTIL.getConfiguration().setBoolean( "hbase.master.enabletable.roundrobin", true); - TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40); TEST_UTIL.startMiniCluster(3); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index 83dd1232dff..236486123bb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -71,7 +71,6 @@ import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.ConnectionUtils; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.NonceGenerator; import org.apache.hadoop.hbase.client.PerClientRandomNonceGenerator; @@ -84,6 +83,7 @@ import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination; import org.apache.hadoop.hbase.exceptions.OperationConflictException; import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException; +import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; @@ -104,11 +104,9 @@ import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; -import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -117,7 +115,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -@Category(LargeTests.class) +@Category({LargeTests.class}) @SuppressWarnings("deprecation") public class TestDistributedLogSplitting { private static final Log LOG = LogFactory.getLog(TestSplitLogManager.class); @@ -168,7 +166,6 @@ public class TestDistributedLogSplitting { conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1); conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, (float) 100.0); // no load balancing conf.setInt("hbase.regionserver.wal.max.splitters", 3); - conf.setInt("hbase.master.maximum.ping.server.attempts", 3); conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40); TEST_UTIL.shutdownMiniHBaseCluster(); TEST_UTIL = new HBaseTestingUtility(conf); @@ -180,7 +177,7 @@ public class TestDistributedLogSplitting { cluster.waitForActiveAndReadyMaster(); master = cluster.getMaster(); while (cluster.getLiveRegionServerThreads().size() < num_rs) { - Threads.sleep(1); + Threads.sleep(10); } } @@ -223,69 +220,75 @@ public class TestDistributedLogSplitting { Path rootdir = FSUtils.getRootDir(conf); - installTable(new ZooKeeperWatcher(conf, "table-creation", null), + Table t = installTable(new ZooKeeperWatcher(conf, "table-creation", null), "table", "family", 40); - TableName table = TableName.valueOf("table"); - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean foundRs = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (HRegionInfo region : regions) { - if (region.getTable().getNameAsString().equalsIgnoreCase("table")) { - foundRs = true; - break; - } - } - if (foundRs) break; - } - final Path logDir = new Path(rootdir, DefaultWALProvider.getWALDirectoryName(hrs - .getServerName().toString())); - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.getTable().getNamespaceAsString() - .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { - it.remove(); - } - } - - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - - slm.splitLogDistributed(logDir); - - int count = 0; - for (HRegionInfo hri : regions) { - - Path tdir = FSUtils.getTableDir(rootdir, table); - Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName())); - LOG.debug("checking edits dir " + editsdir); - FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { - @Override - public boolean accept(Path p) { - if (WALSplitter.isSequenceIdFile(p)) { - return false; + try { + TableName table = t.getName(); + List regions = null; + HRegionServer hrs = null; + for (int i = 0; i < NUM_RS; i++) { + boolean foundRs = false; + hrs = rsts.get(i).getRegionServer(); + regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + for (HRegionInfo region : regions) { + if (region.getTable().getNameAsString().equalsIgnoreCase("table")) { + foundRs = true; + break; } - return true; } - }); - assertTrue("edits dir should have more than a single file in it. instead has " + files.length, - files.length > 1); - for (int i = 0; i < files.length; i++) { - int c = countWAL(files[i].getPath(), fs, conf); - count += c; + if (foundRs) break; } - LOG.info(count + " edits in " + files.length + " recovered edits files."); + final Path logDir = new Path(rootdir, DefaultWALProvider.getWALDirectoryName(hrs + .getServerName().toString())); + + LOG.info("#regions = " + regions.size()); + Iterator it = regions.iterator(); + while (it.hasNext()) { + HRegionInfo region = it.next(); + if (region.getTable().getNamespaceAsString() + .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { + it.remove(); + } + } + + makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); + + slm.splitLogDistributed(logDir); + + int count = 0; + for (HRegionInfo hri : regions) { + + Path tdir = FSUtils.getTableDir(rootdir, table); + Path editsdir = + WALSplitter.getRegionDirRecoveredEditsDir( + HRegion.getRegionDir(tdir, hri.getEncodedName())); + LOG.debug("checking edits dir " + editsdir); + FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { + @Override + public boolean accept(Path p) { + if (WALSplitter.isSequenceIdFile(p)) { + return false; + } + return true; + } + }); + assertTrue( + "edits dir should have more than a single file in it. instead has " + files.length, + files.length > 1); + for (int i = 0; i < files.length; i++) { + int c = countWAL(files[i].getPath(), fs, conf); + count += c; + } + LOG.info(count + " edits in " + files.length + " recovered edits files."); + } + + // check that the log file is moved + assertFalse(fs.exists(logDir)); + + assertEquals(NUM_LOG_LINES, count); + } finally { + if (t != null) t.close(); } - - // check that the log file is moved - assertFalse(fs.exists(logDir)); - - assertEquals(NUM_LOG_LINES, count); } @Test(timeout = 300000) @@ -302,15 +305,17 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + try { + HRegionServer hrs = findRSToKill(false, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - HRegionServer hrs = findRSToKill(false, "table"); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - - // wait for abort completes - this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); - ht.close(); - zkw.close(); + // wait for abort completes + this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); + } } private static class NonceGeneratorWithDups extends PerClientRandomNonceGenerator { @@ -344,10 +349,11 @@ public class TestDistributedLogSplitting { master.balanceSwitch(false); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - HTable ht = installTable(zkw, TABLE_NAME, FAMILY_NAME, NUM_REGIONS_TO_CREATE); + Table ht = installTable(zkw, TABLE_NAME, FAMILY_NAME, NUM_REGIONS_TO_CREATE); NonceGeneratorWithDups ng = new NonceGeneratorWithDups(); NonceGenerator oldNg = - ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection)ht.getConnection(), ng); + ConnectionUtils.injectNonceGeneratorForTesting( + (ClusterConnection)TEST_UTIL.getConnection(), ng); try { List reqs = new ArrayList(); @@ -381,9 +387,10 @@ public class TestDistributedLogSplitting { } } } finally { - ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection) ht.getConnection(), oldNg); - ht.close(); - zkw.close(); + ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection) + TEST_UTIL.getConnection(), oldNg); + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); } } @@ -400,14 +407,16 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + try { + HRegionServer hrs = findRSToKill(true, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - HRegionServer hrs = findRSToKill(true, "table"); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - - this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); - ht.close(); - zkw.close(); + this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); + } } private void abortRSAndVerifyRecovery(HRegionServer hrs, Table ht, final ZooKeeperWatcher zkw, @@ -468,46 +477,47 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + try { + HRegionServer hrs = findRSToKill(false, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - HRegionServer hrs = findRSToKill(false, "table"); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); + // abort master + abortMaster(cluster); - // abort master - abortMaster(cluster); + // abort RS + LOG.info("Aborting region server: " + hrs.getServerName()); + hrs.abort("testing"); - // abort RS - LOG.info("Aborting region server: " + hrs.getServerName()); - hrs.abort("testing"); + // wait for abort completes + TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); + } + }); - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); - } - }); + Thread.sleep(2000); + LOG.info("Current Open Regions:" + + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); - Thread.sleep(2000); - LOG.info("Current Open Regions:" - + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); + // wait for abort completes + TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() + >= (NUM_REGIONS_TO_CREATE + 1)); + } + }); - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() - >= (NUM_REGIONS_TO_CREATE + 1)); - } - }); + LOG.info("Current Open Regions After Master Node Starts Up:" + + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); - LOG.info("Current Open Regions After Master Node Starts Up:" - + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); - - assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); - - ht.close(); - zkw.close(); + assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); + } } @Test(timeout = 300000) @@ -525,50 +535,51 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + try { + HRegionServer hrs = findRSToKill(false, "table"); + List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - HRegionServer hrs = findRSToKill(false, "table"); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); + // abort master + abortMaster(cluster); - // abort master - abortMaster(cluster); + // abort RS + LOG.info("Aborting region server: " + hrs.getServerName()); + hrs.abort("testing"); - // abort RS - LOG.info("Aborting region server: " + hrs.getServerName()); - hrs.abort("testing"); - - // wait for the RS dies - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); - } - }); - - Thread.sleep(2000); - LOG.info("Current Open Regions:" + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); - - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.recoveringRegionsZNode, false); - boolean done = recoveringRegions != null && recoveringRegions.size() == 0; - if (!done) { - LOG.info("Recovering regions: " + recoveringRegions); + // wait for the RS dies + TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); } - return done; - } - }); + }); - LOG.info("Current Open Regions After Master Node Starts Up:" - + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); + Thread.sleep(2000); + LOG.info("Current Open Regions:" + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); - assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); + // wait for all regions are fully recovered + TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( + zkw.recoveringRegionsZNode, false); + boolean done = recoveringRegions != null && recoveringRegions.size() == 0; + if (!done) { + LOG.info("Recovering regions: " + recoveringRegions); + } + return done; + } + }); - ht.close(); - zkw.close(); + LOG.info("Current Open Regions After Master Node Starts Up:" + + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); + + assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); + } } @@ -586,72 +597,74 @@ public class TestDistributedLogSplitting { List rsts = cluster.getLiveRegionServerThreads(); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + try { + List regions = null; + HRegionServer hrs1 = findRSToKill(false, "table"); + regions = ProtobufUtil.getOnlineRegions(hrs1.getRSRpcServices()); - List regions = null; - HRegionServer hrs1 = findRSToKill(false, "table"); - regions = ProtobufUtil.getOnlineRegions(hrs1.getRSRpcServices()); + makeWAL(hrs1, regions, "table", "family", NUM_LOG_LINES, 100); - makeWAL(hrs1, regions, "table", "family", NUM_LOG_LINES, 100); + // abort RS1 + LOG.info("Aborting region server: " + hrs1.getServerName()); + hrs1.abort("testing"); - // abort RS1 - LOG.info("Aborting region server: " + hrs1.getServerName()); - hrs1.abort("testing"); + // wait for abort completes + TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); + } + }); - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); - } - }); + // wait for regions come online + TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() + >= (NUM_REGIONS_TO_CREATE + 1)); + } + }); - // wait for regions come online - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() - >= (NUM_REGIONS_TO_CREATE + 1)); - } - }); + // sleep a little bit in order to interrupt recovering in the middle + Thread.sleep(300); + // abort second region server + rsts = cluster.getLiveRegionServerThreads(); + HRegionServer hrs2 = rsts.get(0).getRegionServer(); + LOG.info("Aborting one more region server: " + hrs2.getServerName()); + hrs2.abort("testing"); - // sleep a little bit in order to interrupt recovering in the middle - Thread.sleep(300); - // abort second region server - rsts = cluster.getLiveRegionServerThreads(); - HRegionServer hrs2 = rsts.get(0).getRegionServer(); - LOG.info("Aborting one more region server: " + hrs2.getServerName()); - hrs2.abort("testing"); + // wait for abort completes + TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 2)); + } + }); - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 2)); - } - }); + // wait for regions come online + TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() + >= (NUM_REGIONS_TO_CREATE + 1)); + } + }); - // wait for regions come online - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() - >= (NUM_REGIONS_TO_CREATE + 1)); - } - }); + // wait for all regions are fully recovered + TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( + zkw.recoveringRegionsZNode, false); + return (recoveringRegions != null && recoveringRegions.size() == 0); + } + }); - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.recoveringRegionsZNode, false); - return (recoveringRegions != null && recoveringRegions.size() == 0); - } - }); - - assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); - ht.close(); - zkw.close(); + assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); + } } @Test(timeout = 300000) @@ -663,41 +676,45 @@ public class TestDistributedLogSplitting { List rsts = cluster.getLiveRegionServerThreads(); final ZooKeeperWatcher zkw = master.getZooKeeper(); Table ht = installTable(zkw, "table", "family", 40); - final SplitLogManager slm = master.getMasterFileSystem().splitLogManager; + try { + final SplitLogManager slm = master.getMasterFileSystem().splitLogManager; - Set regionSet = new HashSet(); - HRegionInfo region = null; - HRegionServer hrs = null; - ServerName firstFailedServer = null; - ServerName secondFailedServer = null; - for (int i = 0; i < NUM_RS; i++) { - hrs = rsts.get(i).getRegionServer(); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - if (regions.isEmpty()) continue; - region = regions.get(0); - regionSet.add(region); - firstFailedServer = hrs.getServerName(); - secondFailedServer = rsts.get((i + 1) % NUM_RS).getRegionServer().getServerName(); - break; - } - - slm.markRegionsRecovering(firstFailedServer, regionSet); - slm.markRegionsRecovering(secondFailedServer, regionSet); - - List recoveringRegions = ZKUtil.listChildrenNoWatch(zkw, - ZKUtil.joinZNode(zkw.recoveringRegionsZNode, region.getEncodedName())); - - assertEquals(recoveringRegions.size(), 2); - - // wait for splitLogWorker to mark them up because there is no WAL files recorded in ZK - final HRegionServer tmphrs = hrs; - TEST_UTIL.waitFor(60000, 1000, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (tmphrs.getRecoveringRegions().size() == 0); + Set regionSet = new HashSet(); + HRegionInfo region = null; + HRegionServer hrs = null; + ServerName firstFailedServer = null; + ServerName secondFailedServer = null; + for (int i = 0; i < NUM_RS; i++) { + hrs = rsts.get(i).getRegionServer(); + List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + if (regions.isEmpty()) continue; + region = regions.get(0); + regionSet.add(region); + firstFailedServer = hrs.getServerName(); + secondFailedServer = rsts.get((i + 1) % NUM_RS).getRegionServer().getServerName(); + break; } - }); - ht.close(); + + slm.markRegionsRecovering(firstFailedServer, regionSet); + slm.markRegionsRecovering(secondFailedServer, regionSet); + + List recoveringRegions = ZKUtil.listChildrenNoWatch(zkw, + ZKUtil.joinZNode(zkw.recoveringRegionsZNode, region.getEncodedName())); + + assertEquals(recoveringRegions.size(), 2); + + // wait for splitLogWorker to mark them up because there is no WAL files recorded in ZK + final HRegionServer tmphrs = hrs; + TEST_UTIL.waitFor(60000, 1000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (tmphrs.getRecoveringRegions().size() == 0); + } + }); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); + } } @Test(timeout = 300000) @@ -712,37 +729,38 @@ public class TestDistributedLogSplitting { List rsts = cluster.getLiveRegionServerThreads(); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; + Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + try { + List regions = null; + HRegionServer hrs = null; + for (int i = 0; i < NUM_RS; i++) { + boolean isCarryingMeta = false; + hrs = rsts.get(i).getRegionServer(); + regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + for (HRegionInfo region : regions) { + if (region.isMetaRegion()) { + isCarryingMeta = true; + break; + } } + if (isCarryingMeta) { + continue; + } + if (regions.size() > 0) break; } - if (isCarryingMeta) { - continue; - } - if (regions.size() > 0) break; + + this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1")); + String originalCheckSum = TEST_UTIL.checksumRows(ht); + + // abort RA and trigger replay + abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); + + assertEquals("Data should remain after reopening of regions", originalCheckSum, + TEST_UTIL.checksumRows(ht)); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); } - - this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1")); - String originalCheckSum = TEST_UTIL.checksumRows(ht); - - // abort RA and trigger replay - abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); - - assertEquals("Data should remain after reopening of regions", originalCheckSum, - TEST_UTIL.checksumRows(ht)); - - ht.close(); - zkw.close(); } @Test(timeout = 300000) @@ -757,134 +775,139 @@ public class TestDistributedLogSplitting { final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table disablingHT = installTable(zkw, "disableTable", "family", NUM_REGIONS_TO_CREATE); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE, NUM_REGIONS_TO_CREATE); + try { + // turn off load balancing to prevent regions from moving around otherwise + // they will consume recovered.edits + master.balanceSwitch(false); - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - List regions = null; - HRegionServer hrs = null; - boolean hasRegionsForBothTables = false; - String tableName = null; - for (int i = 0; i < NUM_RS; i++) { - tableName = null; - hasRegionsForBothTables = false; - boolean isCarryingSystem = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (HRegionInfo region : regions) { - if (region.getTable().isSystemTable()) { - isCarryingSystem = true; - break; - } - if (tableName != null && - !tableName.equalsIgnoreCase(region.getTable().getNameAsString())) { - // make sure that we find a RS has online regions for both "table" and "disableTable" - hasRegionsForBothTables = true; - break; - } else if (tableName == null) { - tableName = region.getTable().getNameAsString(); - } - } - if (isCarryingSystem) { - continue; - } - if (hasRegionsForBothTables) { - break; - } - } - - // make sure we found a good RS - Assert.assertTrue(hasRegionsForBothTables); - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.isMetaTable()) { - it.remove(); - } - } - makeWAL(hrs, regions, "disableTable", "family", NUM_LOG_LINES, 100, false); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - - LOG.info("Disabling table\n"); - TEST_UTIL.getHBaseAdmin().disableTable(TableName.valueOf("disableTable")); - - // abort RS - LOG.info("Aborting region server: " + hrs.getServerName()); - hrs.abort("testing"); - - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); - } - }); - - // wait for regions come online - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() - >= (NUM_REGIONS_TO_CREATE + 1)); - } - }); - - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.recoveringRegionsZNode, false); - ServerManager serverManager = master.getServerManager(); - return (!serverManager.areDeadServersInProgress() && - recoveringRegions != null && recoveringRegions.size() == 0); - } - }); - - int count = 0; - FileSystem fs = master.getMasterFileSystem().getFileSystem(); - Path rootdir = FSUtils.getRootDir(conf); - Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf("disableTable")); - for (HRegionInfo hri : regions) { - Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName())); - LOG.debug("checking edits dir " + editsdir); - if(!fs.exists(editsdir)) continue; - FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { - @Override - public boolean accept(Path p) { - if (WALSplitter.isSequenceIdFile(p)) { - return false; + List regions = null; + HRegionServer hrs = null; + boolean hasRegionsForBothTables = false; + String tableName = null; + for (int i = 0; i < NUM_RS; i++) { + tableName = null; + hasRegionsForBothTables = false; + boolean isCarryingSystem = false; + hrs = rsts.get(i).getRegionServer(); + regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + for (HRegionInfo region : regions) { + if (region.getTable().isSystemTable()) { + isCarryingSystem = true; + break; } - return true; + if (tableName != null && + !tableName.equalsIgnoreCase(region.getTable().getNameAsString())) { + // make sure that we find a RS has online regions for both "table" and "disableTable" + hasRegionsForBothTables = true; + break; + } else if (tableName == null) { + tableName = region.getTable().getNameAsString(); + } + } + if (isCarryingSystem) { + continue; + } + if (hasRegionsForBothTables) { + break; + } + } + + // make sure we found a good RS + Assert.assertTrue(hasRegionsForBothTables); + + LOG.info("#regions = " + regions.size()); + Iterator it = regions.iterator(); + while (it.hasNext()) { + HRegionInfo region = it.next(); + if (region.isMetaTable()) { + it.remove(); + } + } + makeWAL(hrs, regions, "disableTable", "family", NUM_LOG_LINES, 100, false); + makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); + + LOG.info("Disabling table\n"); + TEST_UTIL.getHBaseAdmin().disableTable(TableName.valueOf("disableTable")); + TEST_UTIL.waitTableDisabled(TableName.valueOf("disableTable").getName()); + + // abort RS + LOG.info("Aborting region server: " + hrs.getServerName()); + hrs.abort("testing"); + + // wait for abort completes + TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); } }); - if(files != null) { - for(FileStatus file : files) { - int c = countWAL(file.getPath(), fs, conf); - count += c; - LOG.info(c + " edits in " + file.getPath()); + + // wait for regions come online + TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() + >= (NUM_REGIONS_TO_CREATE + 1)); + } + }); + + // wait for all regions are fully recovered + TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( + zkw.recoveringRegionsZNode, false); + ServerManager serverManager = master.getServerManager(); + return (!serverManager.areDeadServersInProgress() && + recoveringRegions != null && recoveringRegions.size() == 0); + } + }); + + int count = 0; + FileSystem fs = master.getMasterFileSystem().getFileSystem(); + Path rootdir = FSUtils.getRootDir(conf); + Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf("disableTable")); + for (HRegionInfo hri : regions) { + Path editsdir = + WALSplitter.getRegionDirRecoveredEditsDir( + HRegion.getRegionDir(tdir, hri.getEncodedName())); + LOG.debug("checking edits dir " + editsdir); + if(!fs.exists(editsdir)) continue; + FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { + @Override + public boolean accept(Path p) { + if (WALSplitter.isSequenceIdFile(p)) { + return false; + } + return true; + } + }); + if(files != null) { + for(FileStatus file : files) { + int c = countWAL(file.getPath(), fs, conf); + count += c; + LOG.info(c + " edits in " + file.getPath()); + } } } - } - LOG.info("Verify edits in recovered.edits files"); - assertEquals(NUM_LOG_LINES, count); - LOG.info("Verify replayed edits"); - assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); + LOG.info("Verify edits in recovered.edits files"); + assertEquals(NUM_LOG_LINES, count); + LOG.info("Verify replayed edits"); + assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); - // clean up - for (HRegionInfo hri : regions) { - Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName())); - fs.delete(editsdir, true); + // clean up + for (HRegionInfo hri : regions) { + Path editsdir = + WALSplitter.getRegionDirRecoveredEditsDir( + HRegion.getRegionDir(tdir, hri.getEncodedName())); + fs.delete(editsdir, true); + } + disablingHT.close(); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); } - disablingHT.close(); - ht.close(); - zkw.close(); } @Test(timeout = 300000) @@ -902,65 +925,66 @@ public class TestDistributedLogSplitting { List rsts = cluster.getLiveRegionServerThreads(); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - final SplitLogManager slm = master.getMasterFileSystem().splitLogManager; - - Set regionSet = new HashSet(); - HRegionInfo region = null; - HRegionServer hrs = null; - HRegionServer dstRS = null; - for (int i = 0; i < NUM_RS; i++) { - hrs = rsts.get(i).getRegionServer(); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - if (regions.isEmpty()) continue; - region = regions.get(0); - if (region.isMetaRegion()) continue; - regionSet.add(region); - dstRS = rsts.get((i+1) % NUM_RS).getRegionServer(); - break; - } - - slm.markRegionsRecovering(hrs.getServerName(), regionSet); - // move region in order for the region opened in recovering state - final HRegionInfo hri = region; - final HRegionServer tmpRS = dstRS; - TEST_UTIL.getHBaseAdmin().move(region.getEncodedNameAsBytes(), - Bytes.toBytes(dstRS.getServerName().getServerName())); - // wait for region move completes - final RegionStates regionStates = - TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); - TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - ServerName sn = regionStates.getRegionServerOfRegion(hri); - return (sn != null && sn.equals(tmpRS.getServerName())); - } - }); - try { - byte[] key = region.getStartKey(); - if (key == null || key.length == 0) { - key = new byte[] { 0, 0, 0, 0, 1 }; - } - Put put = new Put(key); - put.add(Bytes.toBytes("family"), Bytes.toBytes("c1"), new byte[]{'b'}); - ht.put(put); - } catch (IOException ioe) { - Assert.assertTrue(ioe instanceof RetriesExhaustedWithDetailsException); - RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) ioe; - boolean foundRegionInRecoveryException = false; - for (Throwable t : re.getCauses()) { - if (t instanceof RegionInRecoveryException) { - foundRegionInRecoveryException = true; - break; - } - } - Assert.assertTrue( - "No RegionInRecoveryException. Following exceptions returned=" + re.getCauses(), - foundRegionInRecoveryException); - } + final SplitLogManager slm = master.getMasterFileSystem().splitLogManager; - ht.close(); - zkw.close(); + Set regionSet = new HashSet(); + HRegionInfo region = null; + HRegionServer hrs = null; + HRegionServer dstRS = null; + for (int i = 0; i < NUM_RS; i++) { + hrs = rsts.get(i).getRegionServer(); + List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + if (regions.isEmpty()) continue; + region = regions.get(0); + regionSet.add(region); + dstRS = rsts.get((i+1) % NUM_RS).getRegionServer(); + break; + } + + slm.markRegionsRecovering(hrs.getServerName(), regionSet); + // move region in order for the region opened in recovering state + final HRegionInfo hri = region; + final HRegionServer tmpRS = dstRS; + TEST_UTIL.getHBaseAdmin().move(region.getEncodedNameAsBytes(), + Bytes.toBytes(dstRS.getServerName().getServerName())); + // wait for region move completes + final RegionStates regionStates = + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); + TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + ServerName sn = regionStates.getRegionServerOfRegion(hri); + return (sn != null && sn.equals(tmpRS.getServerName())); + } + }); + + try { + byte[] key = region.getStartKey(); + if (key == null || key.length == 0) { + key = new byte[] { 0, 0, 0, 0, 1 }; + } + Put put = new Put(key); + put.add(Bytes.toBytes("family"), Bytes.toBytes("c1"), new byte[]{'b'}); + ht.put(put); + } catch (IOException ioe) { + Assert.assertTrue(ioe instanceof RetriesExhaustedWithDetailsException); + RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) ioe; + boolean foundRegionInRecoveryException = false; + for (Throwable t : re.getCauses()) { + if (t instanceof RegionInRecoveryException) { + foundRegionInRecoveryException = true; + break; + } + } + Assert.assertTrue( + "No RegionInRecoveryException. Following exceptions returned=" + re.getCauses(), + foundRegionInRecoveryException); + } + } finally { + if (ht != null) ht.close(); + if (ht != null) zkw.close(); + } } /** @@ -986,48 +1010,51 @@ public class TestDistributedLogSplitting { final Path logDir = new Path(rootdir, DefaultWALProvider.getWALDirectoryName(hrs.getServerName().toString())); - installTable(new ZooKeeperWatcher(conf, "table-creation", null), + Table t = installTable(new ZooKeeperWatcher(conf, "table-creation", null), "table", "family", 40); + try { + makeWAL(hrs, ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()), + "table", "family", NUM_LOG_LINES, 100); - makeWAL(hrs, ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()), - "table", "family", NUM_LOG_LINES, 100); - - new Thread() { - @Override - public void run() { - waitForCounter(tot_wkr_task_acquired, 0, 1, 1000); - for (RegionServerThread rst : rsts) { - rst.getRegionServer().abort("testing"); - break; + new Thread() { + @Override + public void run() { + waitForCounter(tot_wkr_task_acquired, 0, 1, 1000); + for (RegionServerThread rst : rsts) { + rst.getRegionServer().abort("testing"); + break; + } + } + }.start(); + // slm.splitLogDistributed(logDir); + FileStatus[] logfiles = fs.listStatus(logDir); + TaskBatch batch = new TaskBatch(); + slm.enqueueSplitTask(logfiles[0].getPath().toString(), batch); + //waitForCounter but for one of the 2 counters + long curt = System.currentTimeMillis(); + long waitTime = 80000; + long endt = curt + waitTime; + while (curt < endt) { + if ((tot_wkr_task_resigned.get() + tot_wkr_task_err.get() + + tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() + + tot_wkr_preempt_task.get()) == 0) { + Thread.yield(); + curt = System.currentTimeMillis(); + } else { + assertTrue(1 <= (tot_wkr_task_resigned.get() + tot_wkr_task_err.get() + + tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() + + tot_wkr_preempt_task.get())); + return; } } - }.start(); - // slm.splitLogDistributed(logDir); - FileStatus[] logfiles = fs.listStatus(logDir); - TaskBatch batch = new TaskBatch(); - slm.enqueueSplitTask(logfiles[0].getPath().toString(), batch); - //waitForCounter but for one of the 2 counters - long curt = System.currentTimeMillis(); - long waitTime = 80000; - long endt = curt + waitTime; - while (curt < endt) { - if ((tot_wkr_task_resigned.get() + tot_wkr_task_err.get() + - tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() + - tot_wkr_preempt_task.get()) == 0) { - Thread.yield(); - curt = System.currentTimeMillis(); - } else { - assertTrue(1 <= (tot_wkr_task_resigned.get() + tot_wkr_task_err.get() + - tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() + - tot_wkr_preempt_task.get())); - return; - } + fail("none of the following counters went up in " + waitTime + + " milliseconds - " + + "tot_wkr_task_resigned, tot_wkr_task_err, " + + "tot_wkr_final_transition_failed, tot_wkr_task_done, " + + "tot_wkr_preempt_task"); + } finally { + if (t != null) t.close(); } - fail("none of the following counters went up in " + waitTime + - " milliseconds - " + - "tot_wkr_task_resigned, tot_wkr_task_err, " + - "tot_wkr_final_transition_failed, tot_wkr_task_done, " + - "tot_wkr_preempt_task"); } @Test (timeout=300000) @@ -1042,46 +1069,49 @@ public class TestDistributedLogSplitting { "distributed log splitting test", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - populateDataInTable(NUM_ROWS_PER_REGION, "family"); + try { + populateDataInTable(NUM_ROWS_PER_REGION, "family"); - List rsts = cluster.getLiveRegionServerThreads(); - assertEquals(NUM_RS, rsts.size()); - rsts.get(0).getRegionServer().abort("testing"); - rsts.get(1).getRegionServer().abort("testing"); - rsts.get(2).getRegionServer().abort("testing"); + List rsts = cluster.getLiveRegionServerThreads(); + assertEquals(NUM_RS, rsts.size()); + rsts.get(0).getRegionServer().abort("testing"); + rsts.get(1).getRegionServer().abort("testing"); + rsts.get(2).getRegionServer().abort("testing"); - long start = EnvironmentEdgeManager.currentTime(); - while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) { - if (EnvironmentEdgeManager.currentTime() - start > 60000) { - assertTrue(false); + long start = EnvironmentEdgeManager.currentTime(); + while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) { + if (EnvironmentEdgeManager.currentTime() - start > 60000) { + assertTrue(false); + } + Thread.sleep(200); } - Thread.sleep(200); + + start = EnvironmentEdgeManager.currentTime(); + while (HBaseTestingUtility.getAllOnlineRegions(cluster).size() + < (NUM_REGIONS_TO_CREATE + 1)) { + if (EnvironmentEdgeManager.currentTime() - start > 60000) { + assertTrue("Timedout", false); + } + Thread.sleep(200); + } + + // wait for all regions are fully recovered + TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( + zkw.recoveringRegionsZNode, false); + return (recoveringRegions != null && recoveringRegions.size() == 0); + } + }); + + assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION, + TEST_UTIL.countRows(ht)); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); } - - start = EnvironmentEdgeManager.currentTime(); - while (HBaseTestingUtility.getAllOnlineRegions(cluster).size() - < (NUM_REGIONS_TO_CREATE + 1)) { - if (EnvironmentEdgeManager.currentTime() - start > 60000) { - assertTrue("Timedout", false); - } - Thread.sleep(200); - } - - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.recoveringRegionsZNode, false); - return (recoveringRegions != null && recoveringRegions.size() == 0); - } - }); - - assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION, - TEST_UTIL.countRows(ht)); - ht.close(); - zkw.close(); } @@ -1218,79 +1248,83 @@ public class TestDistributedLogSplitting { List rsts = cluster.getLiveRegionServerThreads(); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + try { + List regions = null; + HRegionServer hrs = null; + for (int i = 0; i < NUM_RS; i++) { + boolean isCarryingMeta = false; + hrs = rsts.get(i).getRegionServer(); + regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + for (HRegionInfo region : regions) { + if (region.isMetaRegion()) { + isCarryingMeta = true; + break; + } + } + if (isCarryingMeta) { + continue; + } + break; + } - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; + LOG.info("#regions = " + regions.size()); + Iterator it = regions.iterator(); + while (it.hasNext()) { + HRegionInfo region = it.next(); + if (region.isMetaTable() + || region.getEncodedName().equals( + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) { + it.remove(); } } - if (isCarryingMeta) { - continue; + if (regions.size() == 0) return; + HRegionInfo curRegionInfo = regions.get(0); + byte[] startRow = curRegionInfo.getStartKey(); + if (startRow == null || startRow.length == 0) { + startRow = new byte[] { 0, 0, 0, 0, 1 }; } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.isMetaTable() - || region.getEncodedName().equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) { - it.remove(); - } - } - if (regions.size() == 0) return; - HRegionInfo curRegionInfo = regions.get(0); - byte[] startRow = curRegionInfo.getStartKey(); - if (startRow == null || startRow.length == 0) { - startRow = new byte[] { 0, 0, 0, 0, 1 }; - } - byte[] row = Bytes.incrementBytes(startRow, 1); - // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key - row = Arrays.copyOfRange(row, 3, 8); - long value = 0; - TableName tableName = TableName.valueOf("table"); - byte[] family = Bytes.toBytes("family"); - byte[] qualifier = Bytes.toBytes("c1"); - long timeStamp = System.currentTimeMillis(); - HTableDescriptor htd = new HTableDescriptor(); - htd.addFamily(new HColumnDescriptor(family)); - final WAL wal = hrs.getWAL(curRegionInfo); - for (int i = 0; i < NUM_LOG_LINES; i += 1) { - WALEdit e = new WALEdit(); - value++; - e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); - wal.append(htd, curRegionInfo, + byte[] row = Bytes.incrementBytes(startRow, 1); + // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key + row = Arrays.copyOfRange(row, 3, 8); + long value = 0; + TableName tableName = TableName.valueOf("table"); + byte[] family = Bytes.toBytes("family"); + byte[] qualifier = Bytes.toBytes("c1"); + long timeStamp = System.currentTimeMillis(); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(family)); + final WAL wal = hrs.getWAL(curRegionInfo); + for (int i = 0; i < NUM_LOG_LINES; i += 1) { + WALEdit e = new WALEdit(); + value++; + e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); + wal.append(htd, curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis()), - e, sequenceId, true, null); + e, sequenceId, true, null); + } + wal.sync(); + wal.shutdown(); + + // wait for abort completes + this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); + + // verify we got the last value + LOG.info("Verification Starts..."); + Get g = new Get(row); + Result r = ht.get(g); + long theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); + assertEquals(value, theStoredVal); + + // after flush + LOG.info("Verification after flush..."); + TEST_UTIL.getHBaseAdmin().flush(tableName); + r = ht.get(g); + theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); + assertEquals(value, theStoredVal); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); } - wal.sync(); - wal.shutdown(); - - // wait for abort completes - this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); - - // verify we got the last value - LOG.info("Verification Starts..."); - Get g = new Get(row); - Result r = ht.get(g); - long theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); - assertEquals(value, theStoredVal); - - // after flush - LOG.info("Verification after flush..."); - TEST_UTIL.getHBaseAdmin().flush(tableName); - r = ht.get(g); - theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); - assertEquals(value, theStoredVal); - ht.close(); } @Test(timeout = 300000) @@ -1311,88 +1345,91 @@ public class TestDistributedLogSplitting { List rsts = cluster.getLiveRegionServerThreads(); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); + try { + List regions = null; + HRegionServer hrs = null; + for (int i = 0; i < NUM_RS; i++) { + boolean isCarryingMeta = false; + hrs = rsts.get(i).getRegionServer(); + regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + for (HRegionInfo region : regions) { + if (region.isMetaRegion()) { + isCarryingMeta = true; + break; + } + } + if (isCarryingMeta) { + continue; + } + break; + } - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (HRegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; + LOG.info("#regions = " + regions.size()); + Iterator it = regions.iterator(); + while (it.hasNext()) { + HRegionInfo region = it.next(); + if (region.isMetaTable() + || region.getEncodedName().equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) { + it.remove(); } } - if (isCarryingMeta) { - continue; + if (regions.size() == 0) return; + HRegionInfo curRegionInfo = regions.get(0); + byte[] startRow = curRegionInfo.getStartKey(); + if (startRow == null || startRow.length == 0) { + startRow = new byte[] { 0, 0, 0, 0, 1 }; } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - HRegionInfo region = it.next(); - if (region.isMetaTable() - || region.getEncodedName().equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) { - it.remove(); + byte[] row = Bytes.incrementBytes(startRow, 1); + // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key + row = Arrays.copyOfRange(row, 3, 8); + long value = 0; + final TableName tableName = TableName.valueOf("table"); + byte[] family = Bytes.toBytes("family"); + byte[] qualifier = Bytes.toBytes("c1"); + long timeStamp = System.currentTimeMillis(); + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(family)); + final WAL wal = hrs.getWAL(curRegionInfo); + for (int i = 0; i < NUM_LOG_LINES; i += 1) { + WALEdit e = new WALEdit(); + value++; + e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); + wal.append(htd, curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), + tableName, System.currentTimeMillis()), e, sequenceId, true, null); } + wal.sync(); + wal.shutdown(); + + // wait for abort completes + this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); + + // verify we got the last value + LOG.info("Verification Starts..."); + Get g = new Get(row); + Result r = ht.get(g); + long theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); + assertEquals(value, theStoredVal); + + // after flush & compaction + LOG.info("Verification after flush..."); + TEST_UTIL.getHBaseAdmin().flush(tableName); + TEST_UTIL.getHBaseAdmin().compact(tableName); + + // wait for compaction completes + TEST_UTIL.waitFor(30000, 200, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (TEST_UTIL.getHBaseAdmin().getCompactionState(tableName) == CompactionState.NONE); + } + }); + + r = ht.get(g); + theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); + assertEquals(value, theStoredVal); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); } - if (regions.size() == 0) return; - HRegionInfo curRegionInfo = regions.get(0); - byte[] startRow = curRegionInfo.getStartKey(); - if (startRow == null || startRow.length == 0) { - startRow = new byte[] { 0, 0, 0, 0, 1 }; - } - byte[] row = Bytes.incrementBytes(startRow, 1); - // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key - row = Arrays.copyOfRange(row, 3, 8); - long value = 0; - final TableName tableName = TableName.valueOf("table"); - byte[] family = Bytes.toBytes("family"); - byte[] qualifier = Bytes.toBytes("c1"); - long timeStamp = System.currentTimeMillis(); - HTableDescriptor htd = new HTableDescriptor(tableName); - htd.addFamily(new HColumnDescriptor(family)); - final WAL wal = hrs.getWAL(curRegionInfo); - for (int i = 0; i < NUM_LOG_LINES; i += 1) { - WALEdit e = new WALEdit(); - value++; - e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); - wal.append(htd, curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), - tableName, System.currentTimeMillis()), e, sequenceId, true, null); - } - wal.sync(); - wal.shutdown(); - - // wait for abort completes - this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); - - // verify we got the last value - LOG.info("Verification Starts..."); - Get g = new Get(row); - Result r = ht.get(g); - long theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); - assertEquals(value, theStoredVal); - - // after flush & compaction - LOG.info("Verification after flush..."); - TEST_UTIL.getHBaseAdmin().flush(tableName); - TEST_UTIL.getHBaseAdmin().compact(tableName); - - // wait for compaction completes - TEST_UTIL.waitFor(30000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (TEST_UTIL.getHBaseAdmin().getCompactionState(tableName) == CompactionState.NONE); - } - }); - - r = ht.get(g); - theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); - assertEquals(value, theStoredVal); - ht.close(); } @Test(timeout = 300000) @@ -1400,45 +1437,49 @@ public class TestDistributedLogSplitting { LOG.info("testReadWriteSeqIdFiles"); startCluster(2); final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - HTable ht = installTable(zkw, "table", "family", 10); - FileSystem fs = master.getMasterFileSystem().getFileSystem(); - Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), TableName.valueOf("table")); - List regionDirs = FSUtils.getRegionDirs(fs, tableDir); - long newSeqId = WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 1L, 1000L); - WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0) , 1L, 1000L); - assertEquals(newSeqId + 2000, - WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 3L, 1000L)); - - Path editsdir = WALSplitter.getRegionDirRecoveredEditsDir(regionDirs.get(0)); - FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() { - @Override - public boolean accept(Path p) { - return WALSplitter.isSequenceIdFile(p); - } - }); - // only one seqid file should exist - assertEquals(1, files.length); + Table ht = installTable(zkw, "table", "family", 10); + try { + FileSystem fs = master.getMasterFileSystem().getFileSystem(); + Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), TableName.valueOf("table")); + List regionDirs = FSUtils.getRegionDirs(fs, tableDir); + long newSeqId = WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 1L, 1000L); + WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0) , 1L, 1000L); + assertEquals(newSeqId + 2000, + WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 3L, 1000L)); - // verify all seqId files aren't treated as recovered.edits files - NavigableSet recoveredEdits = WALSplitter.getSplitEditFilesSorted(fs, regionDirs.get(0)); - assertEquals(0, recoveredEdits.size()); + Path editsdir = WALSplitter.getRegionDirRecoveredEditsDir(regionDirs.get(0)); + FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() { + @Override + public boolean accept(Path p) { + return WALSplitter.isSequenceIdFile(p); + } + }); + // only one seqid file should exist + assertEquals(1, files.length); - ht.close(); - } - - HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception { + // verify all seqId files aren't treated as recovered.edits files + NavigableSet recoveredEdits = + WALSplitter.getSplitEditFilesSorted(fs, regionDirs.get(0)); + assertEquals(0, recoveredEdits.size()); + } finally { + if (ht != null) ht.close(); + if (zkw != null) zkw.close(); + } + } + + Table installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception { return installTable(zkw, tname, fname, nrs, 0); } - HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs, + Table installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs, int existingRegions) throws Exception { // Create a table with regions TableName table = TableName.valueOf(tname); byte [] family = Bytes.toBytes(fname); LOG.info("Creating table with " + nrs + " regions"); - HTable ht = TEST_UTIL.createMultiRegionTable(table, family, nrs); + Table ht = TEST_UTIL.createMultiRegionTable(table, family, nrs); int numRegions = -1; - try (RegionLocator r = ht.getRegionLocator()) { + try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(table)) { numRegions = r.getStartKeys().length; } assertEquals(nrs, numRegions); @@ -1487,6 +1528,27 @@ public class TestDistributedLogSplitting { putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family); } } + + for (MasterThread mt : cluster.getLiveMasterThreads()) { + HRegionServer hrs = mt.getMaster(); + List hris; + try { + hris = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); + } catch (ServerNotRunningYetException e) { + // It's ok: this master may be a backup. Ignored. + continue; + } + for (HRegionInfo hri : hris) { + if (hri.getTable().isSystemTable()) { + continue; + } + LOG.debug("adding data to rs = " + mt.getName() + + " region = "+ hri.getRegionNameAsString()); + Region region = hrs.getOnlineRegion(hri.getRegionName()); + assertTrue(region != null); + putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family); + } + } } public void makeWAL(HRegionServer hrs, List regions, String tname, String fname, @@ -1544,7 +1606,8 @@ public class TestDistributedLogSplitting { // key byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i)); e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value)); - log.append(htd, curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, + log.append(htd, curRegionInfo, + new HLogKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, System.currentTimeMillis()), e, sequenceId, true, null); if (0 == i % syncEvery) { log.sync(); @@ -1592,10 +1655,8 @@ public class TestDistributedLogSplitting { return count; } - private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master) - throws KeeperException, InterruptedException { - ZKAssign.blockUntilNoRIT(zkw); - master.assignmentManager.waitUntilNoRegionsInTransition(60000); + private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master) throws Exception { + TEST_UTIL.waitUntilNoRegionsInTransition(60000); } private void putData(Region region, byte[] startRow, int numRows, byte [] qf, @@ -1677,14 +1738,19 @@ public class TestDistributedLogSplitting { */ private HRegionServer findRSToKill(boolean hasMetaRegion, String tableName) throws Exception { List rsts = cluster.getLiveRegionServerThreads(); - int numOfRSs = rsts.size(); List regions = null; HRegionServer hrs = null; - for (int i = 0; i < numOfRSs; i++) { + for (RegionServerThread rst: rsts) { + hrs = rst.getRegionServer(); + while (rst.isAlive() && !hrs.isOnline()) { + Thread.sleep(100); + } + if (!rst.isAlive()) { + continue; + } boolean isCarryingMeta = false; boolean foundTableRegion = false; - hrs = rsts.get(i).getRegionServer(); regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); for (HRegionInfo region : regions) { if (region.isMetaRegion()) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java index 1908edba4d4..3258ebdfdaa 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java @@ -69,7 +69,6 @@ public class TestRollingRestart { // Start the cluster log("Starting cluster"); Configuration conf = HBaseConfiguration.create(); - conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40); HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); @@ -237,7 +236,7 @@ public class TestRollingRestart { } return numFound; } - + private void assertRegionsAssigned(MiniHBaseCluster cluster, Set expectedRegions) throws IOException { int numFound = getNumberOfOnlineRegions(cluster); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java index 70bbb690e6d..fab092ead55 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java @@ -111,7 +111,6 @@ public class TestRegionReplicaFailover { conf.setInt("replication.stats.thread.period.seconds", 5); conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, distributedLogReplay); - conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 60); HTU.startMiniCluster(NB_SERVERS); htd = HTU.createTableDescriptor( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java index c23f1c3250a..7e629cdee31 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -81,7 +81,6 @@ public class TestExportSnapshot { conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); conf.setBoolean("hbase.master.enabletable.roundrobin", true); conf.setInt("mapreduce.map.maxattempts", 10); - conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40); } @BeforeClass diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index 34a1b20474b..2d06c0acf5d 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -50,6 +50,14 @@ hbase.regionserver.handler.count 5 + + hbase.regionserver.metahandler.count + 5 + + + hbase.ipc.server.read.threadpool.size + 3 + hbase.master.info.port -1