From dea4aa97ea5f2a6567a3d98ce99114923d89d583 Mon Sep 17 00:00:00 2001 From: Anjan Das Date: Fri, 15 Jan 2021 12:18:12 +0530 Subject: [PATCH] HBASE-25475: Improve UT added as part of HBASE-25445 in TestSplitWALManager (#2855) Signed-off-by: Wellington Chevreuil --- .../hbase/master/TestSplitWALManager.java | 101 ++++++------------ 1 file changed, 32 insertions(+), 69 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java index 10eda749891..40adbeaf9fe 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java @@ -31,14 +31,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure; -import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface; import org.apache.hadoop.hbase.procedure2.Procedure; @@ -98,58 +90,6 @@ public class TestSplitWALManager { TEST_UTIL.shutdownMiniCluster(); } - @Test - public void testWALArchiveWithDifferentWalAndRootFS() throws Exception{ - HBaseTestingUtility test_util_2 = new HBaseTestingUtility(); - Path dir = TEST_UTIL.getDataTestDirOnTestFS("testWalDir"); - test_util_2.getConfiguration().set(CommonFSUtils.HBASE_WAL_DIR, dir.toString()); - CommonFSUtils.setWALRootDir(test_util_2.getConfiguration(), dir); - test_util_2.startMiniCluster(3); - HMaster master2 = test_util_2.getHBaseCluster().getMaster(); - LOG.info("The Master FS is pointing to: " + master2.getMasterFileSystem() - .getFileSystem().getUri()); - LOG.info("The WAL FS is pointing to: " + master2.getMasterFileSystem() - .getWALFileSystem().getUri()); - Table table = test_util_2.createTable(TABLE_NAME, FAMILY); - test_util_2.waitTableAvailable(TABLE_NAME); - Admin admin = test_util_2.getAdmin(); - MasterProcedureEnv env = test_util_2.getMiniHBaseCluster().getMaster() - .getMasterProcedureExecutor().getEnvironment(); - final ProcedureExecutor executor = test_util_2.getMiniHBaseCluster() - .getMaster().getMasterProcedureExecutor(); - List regionInfos = admin.getRegions(TABLE_NAME); - SplitTableRegionProcedure splitProcedure = new SplitTableRegionProcedure( - env, regionInfos.get(0), Bytes.toBytes("row5")); - // Populate some rows in the table - LOG.info("Beginning put data to the table: " + TABLE_NAME.toString()); - int rowCount = 5; - for (int i = 0; i < rowCount; i++) { - byte[] row = Bytes.toBytes("row" + i); - Put put = new Put(row); - put.addColumn(FAMILY, FAMILY, FAMILY); - table.put(put); - } - executor.submitProcedure(splitProcedure); - LOG.info("Submitted SplitProcedure."); - test_util_2.waitFor(30000, () -> executor.getProcedures().stream() - .filter(p -> p instanceof TransitRegionStateProcedure) - .map(p -> (TransitRegionStateProcedure) p) - .anyMatch(p -> TABLE_NAME.equals(p.getTableName()))); - test_util_2.getMiniHBaseCluster().killRegionServer( - test_util_2.getMiniHBaseCluster().getRegionServer(0).getServerName()); - test_util_2.getMiniHBaseCluster().startRegionServer(); - test_util_2.waitUntilNoRegionsInTransition(); - Scan scan = new Scan(); - ResultScanner results = table.getScanner(scan); - int scanRowCount = 0; - while (results.next() != null) { - scanRowCount++; - } - Assert.assertEquals("Got " + scanRowCount + " rows when " + rowCount + - " were expected.", rowCount, scanRowCount); - test_util_2.shutdownMiniCluster(); - } - @Test public void testAcquireAndRelease() throws Exception { List testProcedures = new ArrayList<>(); @@ -272,16 +212,22 @@ public class TestSplitWALManager { Assert.assertEquals(0, metaWals.size()); } - @Test - public void testSplitLogs() throws Exception { - TEST_UTIL.createTable(TABLE_NAME, FAMILY, TEST_UTIL.KEYS_FOR_HBA_CREATE_TABLE); + private void splitLogsTestHelper(HBaseTestingUtility testUtil) throws Exception { + HMaster hmaster = testUtil.getHBaseCluster().getMaster(); + SplitWALManager splitWALManager = hmaster.getSplitWALManager(); + LOG.info("The Master FS is pointing to: " + hmaster.getMasterFileSystem() + .getFileSystem().getUri()); + LOG.info("The WAL FS is pointing to: " + hmaster.getMasterFileSystem() + .getWALFileSystem().getUri()); + + testUtil.createTable(TABLE_NAME, FAMILY, testUtil.KEYS_FOR_HBA_CREATE_TABLE); // load table - TEST_UTIL.loadTable(TEST_UTIL.getConnection().getTable(TABLE_NAME), FAMILY); - ProcedureExecutor masterPE = master.getMasterProcedureExecutor(); - ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta(); - ServerName testServer = TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream() - .map(rs -> rs.getRegionServer().getServerName()).filter(rs -> rs != metaServer).findAny() - .get(); + testUtil.loadTable(testUtil.getConnection().getTable(TABLE_NAME), FAMILY); + ProcedureExecutor masterPE = hmaster.getMasterProcedureExecutor(); + ServerName metaServer = testUtil.getHBaseCluster().getServerHoldingMeta(); + ServerName testServer = testUtil.getHBaseCluster().getRegionServerThreads().stream() + .map(rs -> rs.getRegionServer().getServerName()).filter(rs -> rs != metaServer).findAny() + .get(); List procedures = splitWALManager.splitWALs(testServer, false); Assert.assertEquals(1, procedures.size()); ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0)); @@ -294,6 +240,23 @@ public class TestSplitWALManager { Assert.assertEquals(1, splitWALManager.getWALsToSplit(metaServer, false).size()); } + @Test + public void testSplitLogs() throws Exception { + splitLogsTestHelper(TEST_UTIL); + } + + @Test + public void testSplitLogsWithDifferentWalAndRootFS() throws Exception{ + HBaseTestingUtility testUtil2 = new HBaseTestingUtility(); + testUtil2.getConfiguration().setInt(HBASE_SPLIT_WAL_MAX_SPLITTER, 1); + Path dir = TEST_UTIL.getDataTestDirOnTestFS("testWalDir"); + testUtil2.getConfiguration().set(CommonFSUtils.HBASE_WAL_DIR, dir.toString()); + CommonFSUtils.setWALRootDir(testUtil2.getConfiguration(), dir); + testUtil2.startMiniCluster(3); + splitLogsTestHelper(testUtil2); + testUtil2.shutdownMiniCluster(); + } + @Test public void testWorkerReloadWhenMasterRestart() throws Exception { List testProcedures = new ArrayList<>();